Traefik: do not use dynamic config using docker provider; user hostmode networking

2022-12-08 19:05:03 +00:00 · 2022-12-08 19:05:03 +00:00 · b5b7b70f62
parent 0b00ee06a4
commit b5b7b70f62
5 changed files with 127 additions and 106 deletions
--- a/README.md
+++ b/README.md
@ -1,8 +1,8 @@
 # onyx - Container Server of nbit Informatik GmbH

-In this directory (/home/containers/onyx_pods), you will find all configuration files to run the containers (with Podman and Kubernetes YAML files)
+onyx.nbit.ch is used to run rootless Podman containers, using Traefik as a Reverse Proxy

-Persistent data is stored in /data
+In this directory (/home/containers/onyx_pods), you will find all configuration files to run the containers (with Podman and Kubernetes YAML files)

 Specs:
 - Rocky Linux 9
@ -11,6 +11,8 @@ Specs:
  - 8 GB RAM
  - 80 GB Disk

+Persistent data is stored in /data
+
 ## Create Server

 Name:    onyx.nbit.ch
@ -35,7 +37,11 @@ enable EPEL Repo:
 ```bash
 # dnf install firewalld
 # firewall-cmd --add-service={http,https} --permanent
-# TODO
+# firewall-cmd --remove-service=cockpit --permanent
+# firewall-cmd --reload
+
+List Rules:
+# firewall-cmd  --list-all
 ```

 ## fail2ban on Host for ssh
@ -53,7 +59,6 @@ command to check who is banned:
 # fail2ban-client status sshd
 ```

-
 ## Install Software

 ```bash
@ -126,7 +131,7 @@ Restic Script:
 #
 PATH=$PATH:/usr/local/bin
 export RESTIC_PASSWORD="$(hostname --short)7355"
-restic backup --quiet --repo /backup-restic/restic-repo-$(hostname --short) /home /etc /var /opt /var/lib/docker/volumes /usr/local/bin /backup --exclude=/var/log --exclude=/var/lib/docker/overlay2
+restic backup --quiet --repo /backup-restic/restic-repo-$(hostname --short) /home /etc /var /opt /data /usr/local/bin /backup --exclude=/var/log

 if [ $? -eq 0 ]; then
  restic forget --quiet --repo /backup-restic/restic-repo-$(hostname --short) --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 20 --prune
@ -199,9 +204,50 @@ for dnsserver in ns1.nbit.ch ns2.nbit.ch ; do
 done
 ```

+### Setup Env for Podman
+
+we use /data/<podname> for persitent data:
+```bash
+# mkdir /data
+# chown containers:containers /data
+
+Set Defaults:
+
+containers$ cat ~/.config/containers/containers.conf
+[network]
+network_backend = "netavark"
+```
+
 ### Setup Traefik

-see https://gerov.eu/posts/traefik-for-podman/
+Traefik will be started with podman play kube (with yaml file) and attach to HostNetwork (hostNetwork: true in yaml).
+Backends will map there port and will be accessed on localhost:<port>
+
+we do not use the dynamic config using docker provider, but rather the file provider (one file per backend)
+```bash
+Example Backend Service File:
+
+[containers@onyx onyx_pods]$ cat traefik/configuration/nbitwebsite.yml
+http:
+  routers:
+    nbitwebsite:
+      entrypoints:
+      - websecure
+      tls:
+        certresolver: "myresolver"
+        domains:
+          - main: "www.linux-freelancer.ch"
+            sans: "linux-freelancer.ch"
+      rule: "Host(`linux-freelancer.ch`,`www.linux-freelancer.ch`)"
+      service: nbitwebsite
+
+  services:
+    nbitwebsite:
+      loadBalancer:
+        servers:
+        - url: http://127.0.0.1:9000/
+        passHostHeader: false
+```

 ```bash
 # echo 'net.ipv4.ip_unprivileged_port_start=80' >> /etc/sysctl.d/containers.conf
@ -209,60 +255,29 @@ see https://gerov.eu/posts/traefik-for-podman/
 # The following fixes "Failed to connect to bus: No medium found"
 export XDG_RUNTIME_DIR=/run/user/$(id -u)
 containers$ systemctl --user enable --now podman.socket
-containers$ podman network create web
-containers$ touch acme.json
-containers$ chmod 0600 acme.json
-
-containers$ podman run --name=traefik -d \
-  --security-opt label=type:container_runtime_t \
-  -v /run/user/1000/podman/podman.sock:/var/run/docker.sock:z \
-  -v ./acme.json:/acme.json:z \
-  -v ./configuration/:/configuration/:z \
-  --net web \
-  -p 80:80 \
-  -p 443:443 \
-  --restart always \
-  docker.io/library/traefik:latest \
-  --entrypoints.web.address=":80" \
-  --entrypoints.web.http.redirections.entryPoint.to=websecure \
-  --entrypoints.web.http.redirections.entryPoint.scheme=https \
-  --entrypoints.websecure.address=":443" \
-  --providers.docker=true \
-  --certificatesresolvers.myresolver.acme.email=postmaster@nbit.ch \
-  --certificatesresolvers.myresolver.acme.storage=/acme.json \
-  --certificatesresolvers.myresolver.acme.tlschallenge=true \
-  --certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory \
-  --certificatesresolvers.myresolver.acme.httpChallenge.entrypoint=web \
-  --providers.file.directory=/configuration/ \
-  --providers.file.watch=true \
-  --accesslog=true \
-  --log.level=DEBUG

+containers$ touch /data/traefik/acme.json
+containers$ chmod 0600 /data/traefik/acme.json

 containers$ mkdir -p ~/.config/systemd/user/
 containers$ cd ~/.config/systemd/user/
-containers$ podman generate systemd -f -n traefik
-containers$ systemctl --user enable container-traefik.service
-
-containers$ podman stop traefik
-containers$ systemctl --user start container-traefik.service
-
-# mkdir /data
-# chown containers:containers /data

 containers$ escaped=$(systemd-escape ~/onyx_pods/traefik/traefik.yaml)
 containers$ systemctl --user start podman-kube@$escaped.service
 containers$ systemctl --user enable podman-kube@$escaped.service
-
-
-Hack until Podman supports network in Play YAML:
-
-[containers@onyx default.target.wants]$ pwd
-/home/containers/.config/systemd/user/default.target.wants
-[containers@onyx default.target.wants]$ ls -l
-total 0
-lrwxrwxrwx. 1 containers containers 42 Dec  5 19:16 podman-kube@-home-containers-onyx_pods-nbit_websites-nbit_websites.yaml.service -> /usr/lib/systemd/user/podman-kube@.service
-lrwxrwxrwx. 1 containers containers 42 Dec  3 16:06 podman-kube@-home-containers-onyx_pods-traefik-traefik.yaml.service -> /usr/lib/systemd/user/podman-kube@.service
-
-
+```
+
+### Setup Backend Services
+
+```bash
+Create Kubernetes YAML File:
+
+backendservice=nbitwebsite
+containers$ mkdir ~/onyx_pods/${backendservice}
+
+Create File ~/onyx_pods/${backendservice}/${backendservice}.yaml
+
+containers$ escaped=$(systemd-escape ~/onyx_pods/${backendservice}/${backendservice}.yaml)
+containers$ systemctl --user start podman-kube@$escaped.service
+containers$ systemctl --user enable podman-kube@$escaped.service
 ```
--- a/nbit_websites/nbit_websites.yaml
+++ b/nbit_websites/nbit_websites.yaml
@ -1,41 +0,0 @@
---
-apiVersion: v1
-kind: Pod
-metadata:
-  annotations:
-    bind-mount-options:/data/nbit-website/document_root: z
-    io.kubernetes.cri-o.TTY/nbit_websites: "false"
-    io.podman.annotations.autoremove/nbit_websites: "FALSE"
-    io.podman.annotations.init/nbit_websites: "FALSE"
-    io.podman.annotations.label/nbit_websites: type:container_runtime_t
-    io.podman.annotations.privileged/nbit_websites: "FALSE"
-    io.podman.annotations.publish-all/nbit_websites: "FALSE"
-  labels:
-    app: nbit_websites-pod
-    traefik.enable: true
-    traefik.http.services.nbitwebsite.loadbalancer.server.port: 80
-    traefik.http.routers.nbitwebsite.rule: "Host(`linux-freelancer.ch`,`www.linux-freelancer.ch`)"
-    traefik.http.routers.nbitwebsite.entrypoints: websecure
-    traefik.http.routers.nbitwebsite.tls.certresolver: myresolver
-    traefik.http.routers.nbitwebsite.tls.domains[0].main: www.linux-freelancer.ch
-    traefik.http.routers.nbitwebsite.tls.domains[0].sans: linux-freelancer.ch
-  name: nbit_websites-pod
-spec:
-  containers:
-  - image: docker.io/library/nginx:latest
-    name: nginx
-    securityContext:
-      capabilities:
-        drop:
-        - CAP_MKNOD
-        - CAP_AUDIT_WRITE
-      seLinuxOptions:
-        type: container_runtime_t
-    volumeMounts:
-    - mountPath: /usr/share/nginx/html
-      name: data-nbit-website-host-0
-  volumes:
-  - hostPath:
-      path: /data/nbit-website/document_root
-      type: Directory
-    name: data-nbit-website-host-0
--- a/nbitwebsite/nbitwebsite.yaml
+++ b/nbitwebsite/nbitwebsite.yaml
@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    bind-mount-options:/data/nbitwebsite/document_root: z
+    io.kubernetes.cri-o.TTY/nbitwebsite: "false"
+    io.podman.annotations.autoremove/nbitwebsite: "FALSE"
+    io.podman.annotations.init/nbitwebsite: "FALSE"
+    io.podman.annotations.label/nbitwebsite: type:container_runtime_t
+    io.podman.annotations.privileged/nbitwebsite: "FALSE"
+    io.podman.annotations.publish-all/nbitwebsite: "FALSE"
+  labels:
+    app: nbitwebsite-pod
+  name: nbitwebsite-pod
+spec:
+  containers:
+  - image: docker.io/library/nginx:latest
+    name: nginx
+    ports:
+    - containerPort: 80
+      hostPort: 9000
+      hostIP: 127.0.0.1
+    securityContext:
+      capabilities:
+        drop:
+        - CAP_MKNOD
+        - CAP_AUDIT_WRITE
+      seLinuxOptions:
+        type: container_runtime_t
+    volumeMounts:
+    - mountPath: /usr/share/nginx/html
+      name: data-nbitwebsite-host-0
+  volumes:
+  - hostPath:
+      path: /data/nbitwebsite/document_root
+      type: Directory
+    name: data-nbitwebsite-host-0
--- a/traefik/configuration/nbitwebsite.yml
+++ b/traefik/configuration/nbitwebsite.yml
@ -0,0 +1,19 @@
+http:
+  routers:
+    nbitwebsite:
+      entrypoints:
+      - websecure
+      tls:
+        certresolver: "myresolver"
+        domains:
+          - main: "www.linux-freelancer.ch"
+            sans: "linux-freelancer.ch"
+      rule: "Host(`linux-freelancer.ch`,`www.linux-freelancer.ch`)"
+      service: nbitwebsite
+
+  services:
+    nbitwebsite:
+      loadBalancer:
+        servers:
+        - url: http://127.0.0.1:9000/
+        passHostHeader: false
--- a/traefik/traefik.yaml
+++ b/traefik/traefik.yaml
@ -19,15 +19,13 @@ metadata:
    traefik.http.routers.traefik.tls.domains[0].main: onyx.nbit.ch
  name: traefik-pod
 spec:
+  hostNetwork: true
  containers:
  - args:
    - --entrypoints.web.address=:80
    - --entrypoints.web.http.redirections.entryPoint.to=websecure
    - --entrypoints.web.http.redirections.entryPoint.scheme=https
    - --entrypoints.websecure.address=:443
-    - --providers.docker=true
-    - --providers.docker.exposedbydefault=false
-    - --providers.docker.network=web
    - --certificatesresolvers.myresolver.acme.email=postmaster@nbit.ch
    - --certificatesresolvers.myresolver.acme.storage=/acme.json
    - --certificatesresolvers.myresolver.acme.tlschallenge=true
@ -40,13 +38,6 @@ spec:
    #- --api.insecure=true
    image: docker.io/library/traefik:latest
    name: traefik
-    ports:
-    - containerPort: 80
-      hostPort: 80
-    - containerPort: 443
-      hostPort: 443
-    #- containerPort: 8080
-    #  hostPort: 8080
    securityContext:
      capabilities:
        drop:
@ -74,4 +65,3 @@ spec:
      path: /data/traefik/acme.json
      type: File
    name: data-traefik-acme.json-host-2
-