moby - Container Server

Spezifikaktion:

• Ubuntu Server 20.04
• Hetzner Cloud Server CX31
  • 2 vCPUs
  • 8 GB RAM
  • 80 GB Disk

Erstellen des Servers

Mit dem Binary hcloud von: https://github.com/hetznercloud/cli

Temporaer einen API Key erstellen (nachher wieder loeschen)

$ hcloud context create nbit.ch
$ hcloud image list                          # zeigt moegliche Images
$ hcloud server-type list                    # zeigt moegliche Typen

$ hcloud server create --name moby --image docker-ce --type cx31 --ssh-key joerg@cinnamon.nbit.ch
$ hcloud server set-rdns moby --hostname moby.nbit.ch
$ IPV6="$(hcloud server ip moby -6)"
$ hcloud server set-rdns moby --ip $IPV6 --hostname moby.nbit.ch

DNS Eintraege erstellen:

$ hcloud server ip moby
$ hcloud server ip moby -6                     

# apt update
# apt upgrade

Servername setzen:
# hostnamectl set-hostname moby.nbit.ch

Root-Passwort setzen (das machen wir von Hand)

ssh-Root-Passwort-Login disablen:
/etc/ssh/sshd_config:
PermitRootLogin without-password


Add Swap Space as documented in Mailcow Doc (but we use 2GB):

see https://linuxize.com/post/how-to-add-swap-space-on-ubuntu-20-04/

# fallocate -l 2G /swapfile
# chmod 600 /swapfile
# mkswap /swapfile
# swapon /swapfile
# echo "/swapfile swap swap defaults 0 0" >>/etc/fstab

Firewall

# ufw default deny incoming
# ufw default allow outgoing
# ufw allow ssh
# ufw allow http
# ufw allow https
# ufw allow 1883 # MQTT
# ufw enable

fail2ban auf Host fuer ssh

# apt install fail2ban
# cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
edit /etc/fail2ban/jail.local:
enabled = true unterhalb [sshd]

Check, wer gebanned ist:
# fail2ban-client status sshd

Software installieren

# apt install git

Backup Server

Backup MySQL-DBs:

/usr/local/bin/backup-mysql-dbs.sh (sinngemaess, eine Zeile pro Container):
root@moby:/usr/local/bin# more backup-mysql-dbs.sh 
#!/bin/bash
# Backup der MySQL DBs (Docker)
#
for container_name in $(docker ps --format "{{.Image}} {{.Names}}" |grep mysql  |awk '{print $2}'); do
  if [ -f /usr/local/bin/${container_name}.pwd ]; then
    # im pwd-File muss "PWD=XXXX" (root) gesetzt werden
    . /usr/local/bin/${container_name}.pwd
    docker exec ${container_name} /usr/bin/mysqldump -u root --password=${PWD} --all-databases > /backup/mysql-databases-${container_name}-$(date +%Y%m%W).sql 2>/dev/null
  else
    >&2 echo "Password must be set as PWD=XXXX in /usr/local/bin/${container_name}.pwd" 
  fi
done

# Cleanup Old Backups
find /backup -type f -mtime +30 -exec rm {} \;

/etc/cron.d/backup-mysql-dbs: 
# Backup MySQL DBs
#
45 5 * * * root /usr/local/bin/backup-mysql-dbs.sh >/dev/null


Restore: just in case:
cat backup.sql | docker exec -i CONTAINER /usr/bin/mysql -u root --password=root DATABASE




# apt install restic
# mkdir /backup
# mkdir /backup-restic
# restic init --repo /backup-restic/restic-repo-$(hostname --short)      # Passwort in Keepass

Restic Script:

/usr/local/bin/backup-to-disk.sh
#!/bin/bash
# Backup der wichtigsten Verzeichnisse nach einem Verzeichnis
#
# Es wird restic verwendet.
#
PATH=$PATH:/usr/local/bin
export RESTIC_PASSWORD="$(hostname --short)7355"
restic backup --quiet --repo /backup-restic/restic-repo-$(hostname --short) /home /etc /var /opt /var/lib/docker/volumes /usr/local/bin /backup --exclude=/var/log --exclude=/var/lib/docker/overlay2

if [ $? -eq 0 ]; then
  restic forget --quiet --repo /backup-restic/restic-repo-$(hostname --short) --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 20 --prune
else
  >&2 echo "Problem with restic Backup $(hostname --short)"
fi

/etc/cron.d/backup-to-disk: 
#
# Backup important Files to Disk
#
55 5 * * * root /usr/local/bin/backup-to-disk.sh >/dev/null

Backup auf Storag Box:

# cat > /etc/cron.d/rsync-backup-to-other-host <<HERE
#
# Rsync /backup-restic to backup space
#
20 6 * * * root /usr/bin/rsync -avzH --delete --numeric-ids -e 'ssh -p23' /backup-restic u152662@u152662.your-storagebox.de:moby-backup-restic-rsync >/dev/null
HERE

Systemd Service Unit for docker-compose

see https://community.hetzner.com/tutorials/docker-compose-as-systemd-service

root@moby:/etc/systemd/system# cat docker-compose@.service 
[Unit]
Description=docker-compose %i service
Requires=docker.service network-online.target
After=docker.service network-online.target

[Service]
WorkingDirectory=/home/joerg/moby-configs/%i
Type=simple
TimeoutStartSec=15min
Restart=always
User=joerg
Group=joerg

ExecStartPre=/usr/bin/docker-compose pull --quiet --ignore-pull-failures
ExecStartPre=/usr/bin/docker-compose build --pull

ExecStart=/usr/bin/docker-compose up --remove-orphans --no-color

ExecStop=/usr/bin/docker-compose down --remove-orphans

ExecReload=/usr/bin/docker-compose pull --quiet --ignore-pull-failures
ExecReload=/usr/bin/docker-compose build --pull

[Install]
WantedBy=multi-user.target

# systemctl daemon-reload
# systemctl enable --now docker-compose@proxy
# systemctl enable --now docker-compose@nbit_websites

Logging

root@moby:~# cat /etc/docker/daemon.json
{
	"log-driver": "syslog",
	"log-opts": {
		"syslog-address": "unixgram:///dev/log",
		"tag": "docker/{{.Name}}"
	}
}

root@moby:~# grep -C 3 PreserveFQDN /etc/rsyslog.conf
$IncludeConfig /etc/rsyslog.d/*.conf

# see https://www.commandprompt.com/blog/docker-logging-with-rsyslog/
$PreserveFQDN on

root@moby:~# cat /etc/rsyslog.d/10-docker.conf
$FileCreateMode 0644
$template DockerDaemonLogFileName, "/var/log/docker/docker.log"
$template DockerContainerLogFileName, "/var/log/docker/%SYSLOGTAG:R,ERE,1,FIELD:docker/(.*)\[--end:secpath-replace%.log"
if $programname == 'dockerd'
then {
	?DockerDaemonLogFileName
	stop
}
if $programname == 'containerd'
then {
	?DockerDaemonLogFileName
	stop
}
if $programname == 'docker'
then {
	if $syslogtag contains 'docker/'
	then {
		?DockerContainerLogFileName
		stop
	}
}
$FileCreateMode 0600

root@moby:~# cat /etc/logrotate.d/rsyslog-docker
/var/log/docker/*.log
{
	daily
	rotate 10
	minsize 200M
	missingok
	notifempty
	compress
	sharedscripts
	postrotate
		/usr/lib/rsyslog/rsyslog-rotate
	endscript
}

Wordpress behind Traefik

folgendes muss in wp-config.php eingefuegt werden (ganz oben in PHP Code):

if (strpos($_SERVER['HTTP_X_FORWARDED_PROTO'], 'https') !== false)
   $_SERVER['HTTPS']='on';

DNS Server Tests

root@moby:~# cat /etc/cron.d/checkdnsserver 
#
# Check DNS Server
#
*/15 * * * * root /usr/local/bin/checkdnsserver.sh >/dev/null


root@moby:~# cat /usr/local/bin/checkdnsserver.sh
#!/bin/bash
# 
# Check my DNS servers and report to CloudRadar
#
# Joerg Lehmann, 17.8.2021
#

for dnsserver in ns1.nbit.ch ns2.nbit.ch ; do
  dig +short ${dnsserver} @${dnsserver} >/dev/null 2>&1
  if [ $? -eq 0 ]; then
    csender -t 6tZlIPoM7OQb \
      -u https://hub.cloudradar.io/cct/ \
      -n checkdnsserver \
      -s 1
  else
    csender -t 6tZlIPoM7OQb \
      -u https://hub.cloudradar.io/cct/ \
      -n checkdnsserver \
      -s 0 \
      -a "DNS Test @${dnsserver} failed"
  fi
  sleep 10
done

DNS Slave Server

Dieser Server dient auch als DNS Slave Server (ns2.nbit.ch)

# ufw allow domain
# apt install bind9

add Zones to /etc/bind/named.conf.local:

zone "nbit.ch" IN {
  type slave;
  file "nbit.ch.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "linux-freelancer.ch" IN {
  type slave;
  file "linux-freelancer.ch.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "mini-beieli.ch" IN {
  type slave;
  file "mini-beieli.ch.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "wo-bisch.ch" IN {
  type slave;
  file "wo-bischch.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "ch-wirth.ch" IN {
  type slave;
  file "ch-wirth.ch.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "cmoag.com" IN {
  type slave;
  file "cmoag.com.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};

zone "acmoag.com" IN {
  type slave;
  file "acmoag.com.zone";
  allow-notify { 94.130.184.127; 2a01:4f8:c2c:12ed::1; };
  masters {
    94.130.184.127; 2a01:4f8:c2c:12ed::1;
  };
  allow-transfer {
        127.0.0.1;
  };
};