diff --git a/terraform/.control-plane1.bu.swp b/terraform/.control-plane1.bu.swp deleted file mode 100644 index 7d0e263..0000000 Binary files a/terraform/.control-plane1.bu.swp and /dev/null differ diff --git a/terraform/control-plane1.bu b/terraform/control-plane1.bu index c0866db..ca96a28 100644 --- a/terraform/control-plane1.bu +++ b/terraform/control-plane1.bu @@ -42,7 +42,8 @@ storage: [Network] Address=fd00:0:0:2::91/64 Address=2001:470:7116:2::91/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.91/24 Gateway=10.0.2.3 @@ -65,6 +66,7 @@ storage: 2001:470:7116:2::101 worker1.undercloud.local worker1 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 @@ -72,6 +74,7 @@ storage: fd00:0:0:2::101 worker1.undercloud.local worker1 fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 @@ -79,6 +82,7 @@ storage: 10.0.2.101 worker1.undercloud.local worker1 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -122,6 +126,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -611,4 +616,29 @@ systemd: RestartSec=120s [Install] WantedBy=multi-user.target + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target diff --git a/terraform/control-plane1.bu.mod b/terraform/control-plane1.bu.mod deleted file mode 100644 index bb548c4..0000000 --- a/terraform/control-plane1.bu.mod +++ /dev/null @@ -1,647 +0,0 @@ -variant: flatcar -version: 1.1.0 - -passwd: - users: - - name: core - ssh_authorized_keys: - - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud" - -storage: - directories: - - path: /opt/bin - overwrite: true - mode: 0755 - - path: /opt/cni/bin - overwrite: true - mode: 755 - - path: /etc/kubernetes/manifests - #overwrite: true - mode: 0755 - - path: /etc/install-calico - overwrite: true - mode: 0755 - - path: /var/lib/undercloud-stamps - mode: 0755 - - path: /var/lib/rsyslog - overwrite: true - mode: 0755 - - files: - - path: /etc/hostname - mode: 0644 - contents: - inline: | - control-plane1 - - - path: /etc/systemd/network/00-eth.network - mode: 0644 - contents: - inline: | - [Match] - Name=eth* - - [Network] - Address=fd00:0:0:2::91/64 - Address=2001:470:7116:2::91/64 - Gateway=2001:470:7116:2::3 - DNS=fd00:0:0:1::1 - Address=10.0.2.91/24 - Gateway=10.0.2.3 - DNS=10.0.1.1 - Domains=undercloud.local - IPv6AcceptRA=no - IPv6PrivacyExtensions=no - - - path: /etc/hosts - mode: 0644 - overwrite: true - contents: - inline: | - 127.0.0.1 localhost - ::1 localhost ip6-localhost ip6-loopback - - 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1 - 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2 - 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3 - 2001:470:7116:2::101 worker1.undercloud.local worker1 - 2001:470:7116:2::102 worker2.undercloud.local worker2 - 2001:470:7116:2::103 worker3.undercloud.local worker3 - - fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 - fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 - fd00:0:0:2::93 control-plane3.undercloud.local control-plane3 - fd00:0:0:2::101 worker1.undercloud.local worker1 - fd00:0:0:2::102 worker2.undercloud.local worker2 - fd00:0:0:2::103 worker3.undercloud.local worker3 - - 10.0.2.91 control-plane1.undercloud.local control-plane1 - 10.0.2.92 control-plane2.undercloud.local control-plane2 - 10.0.2.93 control-plane3.undercloud.local control-plane3 - 10.0.2.101 worker1.undercloud.local worker1 - 10.0.2.102 worker2.undercloud.local worker2 - 10.0.2.103 worker3.undercloud.local worker3 - - - path: /etc/motd - mode: 0644 - overwrite: true - contents: - inline: | - ******************************************************************* - * AUTHORIZED ACCESS ONLY * - * * - * This system is part of a secured infrastructure. * - * All activities are monitored and logged. * - * Unauthorized access or misuse is strictly prohibited and * - * may result in disciplinary and legal action. * - ******************************************************************* - - -------------------------------------------------------------------------------- - kubernetes controle plane Node - - Manage via: - kubectl (kubectl) - calico (calicoctl) - velero - backup (velero) - argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/ - -------------------------------------------------------------------------------- - - - path: /etc/sysctl.d/99-k8s.conf - mode: 0644 - contents: - inline: | - net.ipv4.ip_forward = 1 - net.ipv6.ip_forward = 1 - net.ipv6.conf.all.forwarding = 1 - net.ipv4.conf.all.forwarding = 1 - net.bridge.bridge-nf-call-iptables = 1 - net.bridge.bridge-nf-call-ip6tables = 1 - net.netfilter.nf_conntrack_max = 1000000 - net.ipv4.conf.all.rp_filter = 0 - net.ipv6.conf.all.disable_ipv6 = 0 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 524288 - fs.inotify.max_user_instances = 4096 - kernel.panic = 10 - kernel.panic_on_oops = 1 - - - path: /etc/flatcar/update.conf - overwrite: true - mode: 0420 - contents: - inline: | - REBOOT_STRATEGY=off - - - path: /opt/bin/kubeadm - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm" - - - path: /opt/bin/kubelet - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet" - - - path: /opt/bin/kubectl - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl" - - - path: /opt/bin/calicoctl - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl" - - - path: /opt/bin/velero - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero" - - - path: /opt/bin/rsyslogd - mode: 0755 - contents: - source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/rsyslogd" - - - path: /etc/systemd/journald.conf.d/10-forward-to-syslog.conf - mode: 0644 - contents: - inline: | - [Journal] - ForwardToSyslog=yes - Syslog=yes - - - path: /etc/rsyslog.conf - mode: 0644 - contents: - inline: | - # Minimal rsyslog: receive from journald syslog socket and forward to remote - - global(workDirectory="/var/lib/rsyslog") - - # This is the important part: pick up what journald forwards when ForwardToSyslog=yes - module(load="imuxsock" SysSock.Name="/run/systemd/journal/syslog") - - # Forward everything to your syslog server (TCP recommended) - action( - type="omfwd" - target="syslog.undercloud.local" - port="514" - protocol="tcp" - action.resumeRetryCount="-1" - queue.type="linkedList" - queue.size="50000" - ) - - - path: /etc/kubernetes/kubeadm-init.yaml - mode: 0644 - contents: - inline: | - apiVersion: kubeadm.k8s.io/v1beta3 - kind: InitConfiguration - bootstrapTokens: - - token: "kvg1hc.t3rewovrps426rof" - description: "default kubeadm bootstrap token" - ttl: "0" - nodeRegistration: - name: control-plane1 - criSocket: unix:///run/containerd/containerd.sock - kubeletExtraArgs: - node-ip: "2001:470:7116:2::91" - cluster-dns: "10.0.91.53,2001:470:7116:f:1::53" - volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" - localAPIEndpoint: - advertiseAddress: "2001:470:7116:2::91" - bindPort: 6443 - certificateKey: "fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b" - --- - apiVersion: kubeadm.k8s.io/v1beta3 - kind: ClusterConfiguration - controlPlaneEndpoint: "[fd00:0:0:2::100]:6443" - networking: - podSubnet: "2001:470:7116:a::/64,10.0.10.0/24" - serviceSubnet: "2001:470:7116:f:1::/108,10.0.91.0/24" - dnsDomain: "k8s.undercloud.local" - controllerManager: - extraArgs: - flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" - bind-address: '::' - apiServer: - extraArgs: - enable-aggregator-routing: "true" - proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt - proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key - requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt - requestheader-allowed-names: front-proxy-client - requestheader-extra-headers-prefix: X-Remote-Extra- - requestheader-group-headers: X-Remote-Group - requestheader-username-headers: X-Remote-User - - --- - apiVersion: kubelet.config.k8s.io/v1beta1 - kind: KubeletConfiguration - address: "::" - healthzBindAddress: "::" - clusterDomain: "k8s.undercloud.local" - clusterDNS: - - "2001:470:7116:f:1::53" - - "10.0.91.53" - volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec - cgroupDriver: "systemd" - authentication: - anonymous: - enabled: true - webhook: - enabled: true - authorization: - mode: Webhook - - - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml - mode: 0644 - contents: - inline: | - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - spec: - type: ClusterIP - ipFamilyPolicy: RequireDualStack - ipFamilies: [IPv6, IPv4] - clusterIP: 2001:470:7116:f:1::53 - clusterIPs: - - 2001:470:7116:f:1::53 - - 10.0.91.53 - ports: - - name: dns - port: 53 - protocol: UDP - targetPort: 53 - - name: dns-tcp - port: 53 - protocol: TCP - targetPort: 53 - - name: metrics - port: 9153 - protocol: TCP - targetPort: 9153 - selector: - k8s-app: kube-dns - -systemd: - units: - - name: rsyslog.service - enabled: true - contents: | - [Unit] - Description=rsyslog (journald -> remote syslog) - Wants=network-online.target - After=network-online.target systemd-journald.service - Requires=systemd-journald.service - - [Service] - Type=simple - ExecStart=/opt/bin/rsyslogd -n -f /etc/rsyslog.conf - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target - # --- Boot entrypoint: only this target is enabled at boot --- - - name: undercloud-bootstrap.target - enabled: true - contents: | - [Unit] - Description=Undercloud Bootstrap Chain - Wants=network-online.target - After=network-online.target - # Start the chain entry - Wants=containerd.service kubelet.service kubeadm-init.service - After=containerd.service kubelet.service kubeadm-init.service - - [Install] - WantedBy=multi-user.target - - - name: modules-load.service - enabled: true - contents: | - [Unit] - Description=Load necessary kernel modules - Before=containerd.service kubeadm-init.service - - [Service] - Type=oneshot - ExecStart=/usr/bin/modprobe br_netfilter - ExecStart=/usr/bin/modprobe overlay - RemainAfterExit=yes - - [Install] - WantedBy=multi-user.target - - - name: systemd-networkd-wait-online.service - enabled: true - - - name: containerd.service - enabled: true - contents: | - [Unit] - Description=containerd container runtime - After=network.target modules-load.service - Wants=modules-load.service - - [Service] - ExecStart=/usr/bin/containerd - Restart=always - RestartSec=5 - Delegate=yes - KillMode=process - OOMScoreAdjust=-999 - - [Install] - WantedBy=multi-user.target - - - name: set-timezone.service - enabled: true - contents: | - [Unit] - Description=Set Timezone - After=network-online.target - Wants=network-online.target - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"' - ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin - ExecStart=/usr/bin/timedatectl set-ntp true - - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enabled: true - contents: | - [Unit] - Description=kubelet, the Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home - Wants=network-online.target - After=network-online.target containerd.service - Requires=containerd.service - - [Service] - Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" - Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" - EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env - ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS - Restart=always - StartLimitInterval=0 - RestartSec=10 - - [Install] - WantedBy=multi-user.target - - # --- Chain step 1 --- - - name: kubeadm-init.service - enabled: false - contents: | - [Unit] - Description=Kubeadm Init Cluster - Wants=network-online.target - After=network-online.target containerd.service kubelet.service - Requires=containerd.service kubelet.service - ConditionPathExists=!/etc/kubernetes/kubelet.conf - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/ - - ExecStart=/bin/sh -eu -c '\ - echo "[kubeadm-init] started..." ; \ - echo "[kubeadm-init] waiting for containerd socket..." ; \ - for i in $(seq 1 60); do test -S /run/containerd/containerd.sock && break; sleep 1; done ; \ - echo "[kubeadm-init] running kubeadm init..." ; \ - /opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml ; \ - echo "[kubeadm-init] copying kubeconfig to core..." ; \ - mkdir -p /home/core/.kube ; \ - cp -f /etc/kubernetes/admin.conf /home/core/.kube/config ; \ - chown core:core /home/core/.kube/config ; \ - echo "[kubeadm-init] done." \ - ' - - # strictly start next step (serialization) - ExecStartPost=/usr/bin/systemctl start install-calico.service - - [Install] - WantedBy=undercloud-bootstrap.target - - # --- Chain step 2 --- - - name: install-calico.service - enabled: false - contents: | - [Unit] - Description=Install Calico - Requires=kubeadm-init.service - After=kubeadm-init.service - ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - - ExecStart=/bin/sh -eu -c '\ - echo "[calico] waiting for API /readyz..." ; \ - for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ - echo "[calico] create namespace + operator..." ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml ; \ - kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml || true ; \ - kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml || true ; \ - echo "[calico] wait for tigera-operator..." ; \ - kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s ; \ - echo "[calico] apply custom resources..." ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml ; \ - echo "[calico] wait for calico-apiserver..." ; \ - kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s ; \ - echo "[calico] apply peers + pools..." ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml ; \ - echo "[calico] done." \ - ' - - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done - ExecStartPost=/usr/bin/systemctl start install-ceph.service - - [Install] - WantedBy=undercloud-bootstrap.target - - # --- Chain step 3 --- - - name: install-ceph.service - enabled: false - contents: | - [Unit] - Description=Install Ceph CSI - Requires=install-calico.service - After=install-calico.service - ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - - ExecStart=/bin/sh -eu -c '\ - echo "[ceph] waiting for API /readyz..." ; \ - for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ - echo "[ceph] apply manifests..." ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml ; \ - echo "[ceph] done." \ - ' - - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done - ExecStartPost=/usr/bin/systemctl start install-gitea.service - - [Install] - WantedBy=undercloud-bootstrap.target - - # --- Chain step 4 --- - - name: install-gitea.service - enabled: false - contents: | - [Unit] - Description=Install Gitea - Requires=install-ceph.service - After=install-ceph.service - ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - - ExecStart=/bin/sh -eu -c '\ - echo "[gitea] wait for ceph provisioner..." ; \ - kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s ; \ - echo "[gitea] apply manifests..." ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml ; \ - echo "[gitea] wait for gitea deployment..." ; \ - kubectl -n gitea wait deployment gitea --for=condition=Available=True --timeout=1200s ; \ - echo "[gitea] run startup..." ; \ - kubectl exec deploy/gitea -n gitea -- /bin/startup.sh ; \ - echo "[gitea] done." \ - ' - - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done - ExecStartPost=/usr/bin/systemctl start install-argocd.service - - [Install] - WantedBy=undercloud-bootstrap.target - - # --- Chain step 5 --- - - name: install-argocd.service - enabled: false - contents: | - [Unit] - Description=Install ArgoCD - Requires=install-calico.service install-gitea.service - After=install-calico.service install-gitea.service - ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - - ExecStart=/bin/sh -eu -c '\ - echo "[argocd] wait for coredns..." ; \ - kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=1200s ; \ - echo "[argocd] install..." ; \ - kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml ; \ - kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml ; \ - kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=1200s ; \ - kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml ; \ - kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml ; \ - kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml ; \ - echo "[argocd] done." \ - ' - - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done - ExecStartPost=/usr/bin/systemctl start pin-service-ips.service - - [Install] - WantedBy=undercloud-bootstrap.target - - # --- Chain step 6 (final) --- - - name: pin-service-ips.service - enabled: false - contents: | - [Unit] - Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker - Requires=install-argocd.service install-calico.service kubeadm-init.service - After=install-argocd.service install-calico.service kubeadm-init.service network-online.target - Wants=network-online.target - ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done - - [Service] - Type=oneshot - StandardOutput=journal+console - StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=PATH=/usr/bin:/usr/sbin:/opt/bin - ExecStart=/bin/sh -eu -c '\ - echo "[pin-service-ips] waiting for API..." ; \ - for i in $(seq 1 120); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ - echo "[pin-service-ips] ensure namespaces exist..." ; \ - kubectl get ns kube-system >/dev/null ; \ - kubectl get ns argocd >/dev/null 2>&1 || kubectl create ns argocd ; \ - kubectl get ns calico-system >/dev/null ; \ - echo "[pin-service-ips] wait for coredns/argocd readiness (best effort)..." ; \ - kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \ - kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \ - echo "[pin-service-ips] replace Services with fixed ClusterIPs..." ; \ - kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \ - kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \ - kubectl -n argocd delete svc argocd-server --ignore-not-found ; \ - kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \ - kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \ - kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \ - echo "[pin-service-ips] done." \ - ' - - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done - - [Install] - WantedBy=undercloud-bootstrap.target diff --git a/terraform/control-plane2.bu b/terraform/control-plane2.bu index 67c35ce..99ba20b 100644 --- a/terraform/control-plane2.bu +++ b/terraform/control-plane2.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::92/64 Address=2001:470:7116:2::92/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.92/24 Gateway=10.0.2.3 @@ -56,6 +57,7 @@ storage: 2001:470:7116:2::101 worker1.undercloud.local worker1 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 @@ -63,6 +65,7 @@ storage: fd00:0:0:2::101 worker1.undercloud.local worker1 fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 @@ -70,6 +73,7 @@ storage: 10.0.2.101 worker1.undercloud.local worker1 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -113,6 +117,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -292,3 +297,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target \ No newline at end of file diff --git a/terraform/control-plane3.bu b/terraform/control-plane3.bu index 7927368..fc0e941 100644 --- a/terraform/control-plane3.bu +++ b/terraform/control-plane3.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::93/64 Address=2001:470:7116:2::93/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.93/24 Gateway=10.0.2.3 @@ -56,6 +57,7 @@ storage: 2001:470:7116:2::101 worker1.undercloud.local worker1 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 @@ -63,6 +65,7 @@ storage: fd00:0:0:2::101 worker1.undercloud.local worker1 fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 @@ -70,6 +73,7 @@ storage: 10.0.2.101 worker1.undercloud.local worker1 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -113,6 +117,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -292,3 +297,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf index 21b0bb3..09f2d02 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -322,8 +322,8 @@ resource "proxmox_virtual_environment_vm" "control_plane1" { cores = 4 } memory { - dedicated = 6000 - floating = 6000 + dedicated = 8192 + floating = 8192 } # Netzwerk @@ -367,8 +367,8 @@ resource "proxmox_virtual_environment_vm" "control_plane2" { cores = 4 } memory { - dedicated = 4096 - floating = 4096 + dedicated = 8192 + floating = 8192 } # Netzwerk @@ -411,8 +411,8 @@ resource "proxmox_virtual_environment_vm" "control_plane3" { cores = 4 } memory { - dedicated = 4096 - floating = 4096 + dedicated = 8192 + floating = 8192 } # Netzwerk diff --git a/terraform/worker1.bu b/terraform/worker1.bu index 3cdfe11..2f11550 100644 --- a/terraform/worker1.bu +++ b/terraform/worker1.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::101/64 Address=2001:470:7116:2::101/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.101/24 Gateway=10.0.2.3 @@ -54,22 +55,25 @@ storage: 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3 2001:470:7116:2::101 worker1.undercloud.local worker1 - 2001:470:7116:2::102 worker1.undercloud.local worker1 + 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 fd00:0:0:2::93 control-plane3.undercloud.local control-plane3 fd00:0:0:2::101 worker1.undercloud.local worker1 - fd00:0:0:2::102 worker1.undercloud.local worker1 + fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 10.0.2.93 control-plane3.undercloud.local control-plane3 10.0.2.101 worker1.undercloud.local worker1 - 10.0.2.102 worker1.undercloud.local worker1 + 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -109,6 +113,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -271,3 +276,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target diff --git a/terraform/worker2.bu b/terraform/worker2.bu index a4204cf..da4fe4a 100644 --- a/terraform/worker2.bu +++ b/terraform/worker2.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::102/64 Address=2001:470:7116:2::102/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.102/24 Gateway=10.0.2.3 @@ -56,6 +57,7 @@ storage: 2001:470:7116:2::101 worker1.undercloud.local worker1 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 @@ -63,6 +65,7 @@ storage: fd00:0:0:2::101 worker1.undercloud.local worker1 fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 @@ -70,6 +73,7 @@ storage: 10.0.2.101 worker1.undercloud.local worker1 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -109,6 +113,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -271,3 +276,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target diff --git a/terraform/worker3.bu b/terraform/worker3.bu index d0e5657..57c2374 100644 --- a/terraform/worker3.bu +++ b/terraform/worker3.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::103/64 Address=2001:470:7116:2::103/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.103/24 Gateway=10.0.2.3 @@ -56,6 +57,7 @@ storage: 2001:470:7116:2::101 worker1.undercloud.local worker1 2001:470:7116:2::102 worker2.undercloud.local worker2 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 @@ -63,6 +65,7 @@ storage: fd00:0:0:2::101 worker1.undercloud.local worker1 fd00:0:0:2::102 worker2.undercloud.local worker2 fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 10.0.2.91 control-plane1.undercloud.local control-plane1 10.0.2.92 control-plane2.undercloud.local control-plane2 @@ -70,6 +73,7 @@ storage: 10.0.2.101 worker1.undercloud.local worker1 10.0.2.102 worker2.undercloud.local worker2 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 - path: /etc/motd mode: 0644 @@ -109,6 +113,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -271,3 +276,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target diff --git a/terraform/worker4.bu b/terraform/worker4.bu index 1c459d4..7807351 100644 --- a/terraform/worker4.bu +++ b/terraform/worker4.bu @@ -33,7 +33,8 @@ storage: [Network] Address=fd00:0:0:2::104/64 Address=2001:470:7116:2::104/64 - Gateway=2001:470:7116:2::3 + #Gateway=2001:470:7116:2::3 + Gateway=fd00:0:0:2::3 DNS=fd00:0:0:1::1 Address=10.0.2.104/24 Gateway=10.0.2.3 @@ -112,6 +113,7 @@ storage: fs.inotify.max_user_instances = 4096 kernel.panic = 10 kernel.panic_on_oops = 1 + net.ipv6.conf.all.accept_ra = 2 - path: /etc/flatcar/update.conf overwrite: true @@ -274,3 +276,30 @@ systemd: [Install] WantedBy=multi-user.target + + - name: ping-all.service + enabled: true + contents: | + [Unit] + Description=Ping cluster nodes every 5 seconds + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/bin/bash -c '\ + while true; do \ + ping -c1 -W1 fd00:0:0:2::91; \ + ping -c1 -W1 fd00:0:0:2::92; \ + ping -c1 -W1 fd00:0:0:2::93; \ + ping -c1 -W1 fd00:0:0:2::101; \ + ping -c1 -W1 fd00:0:0:2::102; \ + ping -c1 -W1 fd00:0:0:2::103; \ + ping -c1 -W1 fd00:0:0:2::104; \ + sleep 5; \ + done' + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target