paulgorman.org/technical

FreeBSD Blinky Runbook

Blinky is a home server and firewall running FreeBSD that I build in late 2015.

Components

FreeBSD Installation

FreeBSD Configuration

# pkg install sudo vim-lite zsh git-lite sshguard-ipfw curl tmux cowsay ddclient weechat isc-dhcp43-server vnstat iocage haproxy alpine
# ssh-keygen -t rsa -b 4096 -C "paulgorman@blinky.example.com"

Edit /etc/sysctl.conf:

vfs.zfs.l2arc_noprefetch=0
net.inet.ip.fw.verbose_limit=5

Edit /etc/rc.conf:

hostname="blinky.example.com"
zfs_enable="YES"
ifconfig_em1="inet 10.0.1.1 netmask 255.255.255.0"
ifconfig_em0="DHCP"
sshd_enable="YES"
sshguard_enable="YES"
sshguard_safety_thresh="30"
sshguard_pardon_min_interval="3600"
sshguard_prescribe_interval="7200"
ntpd_enable="YES"
ntpd_sync_on_start="YES"
ddclient_enable="YES"
gateway_enable="YES"
firewall_enable="YES"
firewall_script="/etc/ipfw.rules"
firewall_nat_enable="YES"
firewall_nat_interface="em0"
firewall_logging="YES"
cloned_interfaces="lo1"
ifconfig_lo1="inet 172.16.0.1/24"
wlans_ath0="wlan0"
create_args_wlan0="wlanmode hostap"
ifconfig_wlan0="inet 10.0.2.1 netmask 0xffffff00 ssid foonet mode 11g channel 2"
hostapd_enable="YES"
dhcpd_enable="YES"                          # dhcpd enabled?
dhcpd_flags="-q"                            # command option(s)
dhcpd_conf="/usr/local/etc/dhcpd.conf"      # configuration file
dhcpd_ifaces="em1 wlan0"                    # ethernet interface(s)
dhcpd_withumask="022"                       # file creation mask
apcupsd_enable="YES"
vnstat_enable="YES"
iocage_enable="YES"
haproxy_enable="YES"
kdc_enable="YES"
kadmind_enable="YES"
gssd_enable="YES"
nfs_server_enable="YES"
nfsv4_server_enable="YES"
nfsuserd_enable="YES"

Edit /boot/loader.conf:

geom_mirror_load="YES"
kern.geom.label.gptid.enable="0"
zfs_load="YES"
vfs.zfs.arc_max="5120M"
if_ath_load="YES"
wlan_wep_load="YES"
wlan_ccmp_load="YES"
wlan_tkip_load="YES"

Edit /etc/ipfw.rules:

ipfw -q -f flush
add="ipfw -q add"
wan="em0"
lan="em1"
wifi="wlan0"
jails="lo1"
natout="60000"
tcpout="https,http,domain,ssh,ntp,43,67,68,3478,3479,3480,5223,6665,6666,6667,6697,7000,7070,8080,8880,44422"
udpout="domain,ntp,67,68,1194,3478,3479"
# Playstation3: tcp 3478,3479,3480,5223,8080; udp 3478,3479

ipfw -q nat 1 config if $wan reset unreg_only same_ports

$add 10 allow all from any to any via $lan
$add 20 allow all from any to any via $wifi
$add 30 allow all from any to any via lo0
$add 40 allow all from any to any via $jails

$add 100 deny all from any to 127.0.0.0/8
$add 110 deny ip from 127.0.0.0/8 to any
$add 120 deny all from any to ::1
$add 130 deny all from ::1 to any

$add 300 deny all from any to any not antispoof in
$add 310 deny all from any to 10.0.0.0/8 via $wan
$add 320 deny all from any to 172.16.0.0/12 via $wan
$add 330 deny all from any to 192.168.0.0/16 via $wan
$add 340 deny all from any to 0.0.0.0/8 via $wan
$add 350 deny all from any to 169.254.0.0/16 via $wan
$add 360 deny all from any to 224.0.0.0/4 via $wan
$add 370 deny all from any to 240.0.0.0/4 via $wan

$add 1000 allow tcp from any to me http,https setup keep-state

$add 10000 deny ip from table\(22\) to any

#### Allow ssh after rules created by sshguard (55000 range).
#### Additional note: the note above is no long accurate as rules are added to table 22.
$add 56000 allow tcp from any to me ssh setup keep-state

$add 57000 nat 1 all from any to any in recv $wan
$add 57001 check-state

$add 58000 skipto $natout tcp from any to any $tcpout out xmit $wan keep-state
$add 58010 skipto $natout udp from any to any $udpout out xmit $wan keep-state
$add 58020 skipto $natout icmp from any to any out xmit $wan keep-state

$add 59999 deny log all from any to any

$add 60000 nat 1 all from any to any out xmit $wan
$add 60001 allow all from any to any

Edit /usr/local/etc/vnstat.conf:

Interface "em0"

Edit /etc/hostapd.conf:

interface=wlan0
debug=1
ctrl_interface=/var/run/hostapd
ctrl_interface_group=wheel
ssid=foonet
wpa=2
wpa_passphrase=**********************
wpa_key_mgmt=WPA-PSK
wpa_pairwise=CCMP

Edit /usr/local/etc/dhcpd.conf

# option definitions common to all supported networks...
option domain-name "example.com";
option domain-name-servers 8.8.8.8, 75.75.75.75;
option subnet-mask 255.255.255.0;

default-lease-time 600;
max-lease-time 7200;

# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
authoritative;

# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
log-facility local7;

subnet 10.0.1.0 netmask 255.255.255.0 {
  range 10.0.1.101 10.0.1.199;
  option routers 10.0.1.1;
}

subnet 10.0.2.0 netmask 255.255.255.0 {
  range 10.0.2.101 10.0.2.199;
  option routers 10.0.2.1;
}

Edit /usr/local/etc/ddclient.conf:

daemon=900    # check every 900 seconds
syslog=yes    # log update msgs to syslog
mail-failure=root    # mail failed update msgs to root
pid=/var/run/ddclient.pid    # record PID in file.
ssl=yes    # use ssl-support. Works with ssl-library
use=web, web=dynamicdns.park-your-domain.com/getip
protocol=namecheap
server=dynamicdns.park-your-domain.com
login=example.com
password=***********************************
blinky

Edit /etc/periodic.conf:

daily_ddclient_force_enable="YES"

Create fallback ip-address-to-dreamhost.sh in ~/bin:

#!/bin/tcsh
/usr/bin/host myip.opendns.com resolver1.opendns.com | /usr/bin/grep "has address" | /usr/bin/awk '{print $NF}' > /tmp/ip.txt
/usr/bin/scp -q /tmp/ip.txt example.org:w/

Make it executable, and set a cron job:

@hourly $HOME/bin/ip-address-to-dreamhost.sh

Creating the ZFS data storage pool

dmesg | grep "^ad" shows our 1 TB hard drives are ada1 and ada2.

# gpart create -s GPT ada1
# gpart create -s GPT ada2
# gpart add -a 4k -s 931G -t freebsd-zfs ada1
# gpart add -a 4k -s 931G -t freebsd-zfs ada2
# gpart show
# zpool create data mirror /dev/ada1p1 /dev/ada2p1
# zpool status
# ls /data
# echo 'daily_status_zfs_enable="YES"' >> /etc/periodic.conf
# echo 'daily_scrub_zfs_enable="YES"' >> /etc/periodic.conf

(periodic.conf(5) says ‘daily_scrub_zfs_enable’ does a scrub every thirty-five days by default.)

Use our final SSD (ada0) as L2ARC and ZIL for the spinning disks:

# gpart create -s GPT ada0
# gpart add -a 4k -s 100G -t freebsd-zfs -l L2ARC ada0
# gpart add -a 4k -s 8G -t freebsd-zfs -l ZIL ada0
# zpool add data cache /dev/ada0p1
# zpool add data log /dev/ada0p2
# sysctl vfs.zfs.l2arc_noprefetch=0
# echo 'vfs.zfs.l2arc_noprefetch=0' >> /etc/sysctl.conf

(We turn off l2arc_noprefetch because the throughput of our SSD cache is probably greater than that of the pool for sequential reads.)

File Share (sshfs)

Create a file share:

# zfs create data/share
# chmod g+rwx /data/share/

(Our user is part of the wheel group.)

On our remote client:

# apt-get install sshfs
% mkdir ~/blinky
% sshfs -p 22 blinky.example.com:/data/share ~/blinky

Set local mail delivery

Edit /etc/mail/aliases and add:

root: paulgorman

Run newaliases to update the aliases database.

(If we later get outgoing mail, we’ll change the address.)

Secure ssh

Once we’ve added keys for our usual remote workstations to ~/.ssh/authorized_keys, turn off password-based logins. In /etc/ssh/sshd_config:

PermitRootLogin no
PasswordAuthentication no

Configure UPS

# pkg install apcupsd

Edit /usr/local/etc/apcupsd/apcupsd.conf:

## apcupsd.conf v1.1 ##
# apcupsd must be restarted for changes to this file to become active.
# ========= General configuration parameters ============
UPSNAME APCBU550
UPSCABLE usb
UPSTYPE usb
DEVICE
LOCKFILE /var/spool/lock
SCRIPTDIR /usr/local/etc/apcupsd
PWRFAILDIR /var/run
NOLOGINDIR /var/run
# ======== Configuration parameters used during power failures ==========
# The ONBATTERYDELAY is the time in seconds from when a power failure
#   is detected until we react to it with an onbattery event.
ONBATTERYDELAY 6
#
# Note: BATTERYLEVEL, MINUTES, and TIMEOUT work in conjunction, so
# the first that occurs will cause the initation of a shutdown.
#
# The remaining battery percentage apcupsd will initiate a shutdown.
BATTERYLEVEL 10
# The remaining runtime in minutes apcupsd will initiate a shutdown.
MINUTES 3
# If during a power failure, the UPS has run on batteries for TIMEOUT
# many seconds or longer, apcupsd will initiate a system shutdown.
# A value of 0 disables this timer.
TIMEOUT 120
#  Seconds between user signoff messages prior to shutdown. 0 disables.
ANNOY 300
ANNOYDELAY 60
NOLOGON disable
KILLDELAY 0
# ==== Configuration statements for Network Information Server ====
NETSERVER off
NISIP 0.0.0.0
NISPORT 3551
EVENTSFILE /var/log/apcupsd.events
EVENTSFILEMAX 10
# ========== Configuration statements if sharing =============
#            a UPS with more than one machine
UPSCLASS standalone
UPSMODE disable
# ===== Configuration statements to control apcupsd system logging ========
STATTIME 0
STATFILE /var/log/apcupsd.status
LOGSTATS off
DATATIME 0

iocage

# iocage activate data
# iocage fetch
# iocage create -c tag=myjail ip4_address="lo1|172.16.0.2/24"

haproxy

Edit /usr/local/etc/haproxy.conf:

global
    daemon
    maxconn 1024

defaults
        mode http
        timeout connect 5000ms
        timeout client 50000ms
        timeout server 50000ms

frontend http
        bind *:80
        default_backend www-dashboard
        acl www-dashboard hdr(host) -i blinky.example.com
        use_backend www-dashboard if www-dashboard

backend www-dashboard
        server test1 172.16.0.5:80

NFSv4

We’ll serve files over NFSv4, and set up Kerberos to do so.

(Note: running a kdc on a box that does a bunch of other stuff is normally bad for security. In this case, we’re only using it for nfs, so compromising this kdc doesn’t gain an attacker anything extra.)

Edit /etc/krb5.conf:

[libdefaults]
    default_realm = EXAMPLE.ORG
[realms]
    EXAMPLE.ORG = {
        kdc = kerberos.example.org
        admin_server = kerberos.example.org
    }
[domain_realm]
    .example.org = EXAMPLE.ORG

Create the Kerberos database and add a principal by running:

# kstash
Master key: xxxxxxxxxxxxxxxxxxxxxxx
Verifying password - Master key: xxxxxxxxxxxxxxxxxxxxxxx
kadmin> add myprincipal
Max ticket life [unlimited]:
Max renewable life [unlimited]:
Attributes []:
Password: xxxxxxxx
Verifying password - Password: xxxxxxxx

Supply a long and strong password. Because it’s stored in /var/heimdal/m-key, it’s not necessary to remember the password.

Initialize the database:

# kadmin -l
kadmin> init EXAMPLE.ORG
Realm max ticket life [unlimited]:

Start the daemons:

# service kdc start
# service kadmind start

On the Debian client:

# apt-get install nfs-common heimdal-clients

On the Debian client, edit /etc/default/nfs-common:

NEED_IDMAPD=yes
NEED_GSSD=yes

Test by obtaining a ticket on our client, and viewing it:

$ kinit myprincipal
$ klist

Back to the server, we create kerberos principals for the nfs server and client, and generate keytab files for them:

# kadmin -l
kadmin> add -r nfs/blinky.example.com
kadmin> add -r nfs/client.example.com
kadmin> ext_keytab -k /etc/krb5.keytab nfs/blinky.example.com
kadmin> ext_keytab -k /tmp/client.keytab nfs/client.example.com

scp the client.keytab to our Debian client, and on the client:

# mv client.keytab /etc/krb5.keytab
# chown root:root /etc/krb5.keytab
# chmod 600 /etc/krb5.keytab

(If the client or server already has a keytab file, merge the new key into the existing file: ktutil copy new.keytab /etc/krb5.keytab and then verify the keys with ktutil -k /etc/krb5.keytab list.)