remote-syslog

terraform/control-plane1.bu

@@ -154,6 +154,11 @@ storage:
       contents:
         source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero"
     
+    - path: /opt/bin/remote-syslog
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/remote-syslog"
+
     - path: /etc/kubernetes/kubeadm-init.yaml
       mode: 0644
       contents:
@@ -209,7 +214,6 @@ storage:
             - "10.0.91.53"
           volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec
           cgroupDriver: "systemd"
-          apiVersion: kubelet.config.k8s.io/v1beta1
           authentication:
             anonymous:
               enabled: true
@@ -253,9 +257,23 @@ storage:
             selector:
               k8s-app: kube-dns
 
-
 systemd:
   units:
+    # --- Boot entrypoint: only this target is enabled at boot ---
+    - name: undercloud-bootstrap.target
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Undercloud Bootstrap Chain
+        Wants=network-online.target
+        After=network-online.target
+        # Start the chain entry
+        Wants=containerd.service kubelet.service kubeadm-init.service
+        After=containerd.service kubelet.service kubeadm-init.service
+
+        [Install]
+        WantedBy=multi-user.target
+
     - name: modules-load.service
       enabled: true
       contents: |
@@ -265,7 +283,6 @@ systemd:
 
         [Service]
         Type=oneshot
-
         ExecStart=/usr/bin/modprobe br_netfilter
         ExecStart=/usr/bin/modprobe overlay
         RemainAfterExit=yes
@@ -282,11 +299,9 @@ systemd:
         [Unit]
         Description=containerd container runtime
         After=network.target modules-load.service
+        Wants=modules-load.service
 
         [Service]
-        #StandardOutput=journal+console
-        #StandardError=journal+console
-
         ExecStart=/usr/bin/containerd
         Restart=always
         RestartSec=5
@@ -304,17 +319,15 @@ systemd:
         Description=Set Timezone
         After=network-online.target
         Wants=network-online.target
-        [Service]
-        StandardOutput=journal+console
-        StandardError=journal+console
 
-        ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"'
+        [Service]
+        Type=oneshot
         StandardOutput=journal+console
         StandardError=journal+console
-        Type=oneshot
+        ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"'
-        Restart=on-failure
         ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin
         ExecStart=/usr/bin/timedatectl set-ntp true
+
         [Install]
         WantedBy=multi-user.target
 
@@ -323,32 +336,32 @@ systemd:
       contents: |
         [Unit]
         Description=kubelet, the Kubernetes Node Agent
-        Documentation=https://kubernets.io/docs/home
+        Documentation=https://kubernetes.io/docs/home
         Wants=network-online.target
-        After=network-online.target
+        After=network-online.target containerd.service
-        [Service]
+        Requires=containerd.service
-        #StandardOutput=journal+console
-        #StandardError=journal+console
 
-        #EnvironmentFile=/run/metadata/coreos
+        [Service]
         Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
         Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
-        # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
         EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
         ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
         Restart=always
         StartLimitInterval=0
         RestartSec=10
+
         [Install]
         WantedBy=multi-user.target
 
+    # --- Chain step 1 ---
     - name: kubeadm-init.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
         Description=Kubeadm Init Cluster
-        After=network-online.target containerd.service kubelet.service
         Wants=network-online.target
+        After=network-online.target containerd.service kubelet.service
+        Requires=containerd.service kubelet.service
         ConditionPathExists=!/etc/kubernetes/kubelet.conf
 
         [Service]
@@ -356,230 +369,202 @@ systemd:
         StandardOutput=journal+console
         StandardError=journal+console
 
-        ExecStart=/bin/sh -c 'echo "kubeadm-init.service started..."'
-
-        # Environment
         Environment=KUBECONFIG=/etc/kubernetes/admin.conf
         Environment=DATASTORE_TYPE=kubernetes
         Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/
 
-        ExecStartPre=/bin/sleep 30s
+        ExecStart=/bin/sh -eu -c '\
-        ExecStart=/bin/sh -c 'echo "running kubeadm init..."'
+          echo "[kubeadm-init] started..." ; \
-        ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml
+          echo "[kubeadm-init] waiting for containerd socket..." ; \
+          for i in $(seq 1 60); do test -S /run/containerd/containerd.sock && break; sleep 1; done ; \
+          echo "[kubeadm-init] running kubeadm init..." ; \
+          /opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml ; \
+          echo "[kubeadm-init] copying kubeconfig to core..." ; \
+          mkdir -p /home/core/.kube ; \
+          cp -f /etc/kubernetes/admin.conf /home/core/.kube/config ; \
+          chown core:core /home/core/.kube/config ; \
+          echo "[kubeadm-init] done." \
+        '
 
-        # copy files for kubectl
+        # strictly start next step (serialization)
-        ExecStart=/bin/sh -c 'echo "copying files (admin.conf) to core home folder."'
+        ExecStartPost=/usr/bin/systemctl start install-calico.service
-        ExecStartPost=/usr/bin/mkdir -p /home/core/.kube
-        ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config
-        ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config
 
-        ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service
-        Restart=on-failure
-        RestartSec=120s
         [Install]
-        WantedBy=multi-user.target
+        WantedBy=undercloud-bootstrap.target
+
+    # --- Chain step 2 ---
     - name: install-calico.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
-        Wants=kubeadm-init.service
+        Description=Install Calico
+        Requires=kubeadm-init.service
         After=kubeadm-init.service
         ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done
 
-
         [Service]
-        StandardOutput=journal+console
-        StandardError=journal+console
-
-        ExecStart=/bin/sh -c 'echo "install.calico.service started..."'
-        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
-        Environment=DATASTORE_TYPE=kubernetes
-        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
         Type=oneshot
         StandardOutput=journal+console
         StandardError=journal+console
-        ExecStart=/bin/sh -c 'echo "witing 30s..."'
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
-        ExecStart=/bin/sleep 30s
+        Environment=DATASTORE_TYPE=kubernetes
-        ExecStart=/bin/sh -c 'echo "create calico namespace..."'
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml
-        ExecStart=/bin/sh -c 'echo "install tigera operator..."'
-        ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml
-        ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml
-        ExecStart=/bin/sh -c 'echo "witing 60s..."'
-        ExecStart=/bin/sleep 60s
-        ExecStart=/bin/sh -c 'echo "witing for tigera operator... (20mini max)"'
-        ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s
-        ExecStart=/bin/sh -c 'echo "create clico custom ressources..."'
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml
 
-        ExecStart=/bin/sh -c 'echo "witing 3m.."'
+        ExecStart=/bin/sh -eu -c '\
-        ExecStart=/bin/sleep 3m
+          echo "[calico] waiting for API /readyz..." ; \
-        #ExecStart=/bin/sh -c 'echo "apply calico (calico-apiserver)..."'
+          for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \
-        #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml
+          echo "[calico] create namespace + operator..." ; \
-        #ExecStart=/bin/sh -c 'echo "witing 1m..."'
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml ; \
-        #ExecStart=/bin/sleep 2m
+          kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml || true ; \
-        ExecStart=/bin/sh -c 'echo "witing calico-apiserver... (20mini max)"'
+          kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml || true ; \
-        ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s
+          echo "[calico] wait for tigera-operator..." ; \
-        ExecStart=/bin/sh -c 'echo "witing 120s..."'
+          kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s ; \
-        ExecStart=/bin/sleep 2m
+          echo "[calico] apply custom resources..." ; \
-        ExecStart=/bin/sh -c 'echo "apply calico-peers..."'
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml
+          echo "[calico] wait for calico-apiserver..." ; \
-        ExecStart=/bin/sh -c 'echo "witing 60s..."'
+          kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s ; \
-        ExecStart=/bin/sleep 1m
+          echo "[calico] apply peers + pools..." ; \
-        ExecStart=/bin/sh -c 'echo "apply calico-ippools..."'
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml ; \
-        
+          echo "[calico] done." \
-        #ExecStart=/bin/sh -c 'echo "witing for whisker.."'
+        '
-        #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s
-        #ExecStart=/bin/sh -c 'echo "port-forward -n calico-system service/whisker 8081:8081"'
-        #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081
 
         ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done
-        ExecStart=/usr/bin/systemctl disable install-calico.service
+        ExecStartPost=/usr/bin/systemctl start install-ceph.service
-        #RemainAfterExit=true
+
-        Restart=on-failure
-        RestartSec=120s
         [Install]
-        WantedBy=multi-user.target
+        WantedBy=undercloud-bootstrap.target
+
+    # --- Chain step 3 ---
     - name: install-ceph.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
-        Wants=kubeadm-init.service
+        Description=Install Ceph CSI
-        After=kubeadm-init.service
+        Requires=install-calico.service
+        After=install-calico.service
         ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done
 
         [Service]
+        Type=oneshot
         StandardOutput=journal+console
         StandardError=journal+console
-
-        ExecStart=/bin/sh -c 'echo "install.ceph.service started..."'
         Environment=KUBECONFIG=/etc/kubernetes/admin.conf
         Environment=DATASTORE_TYPE=kubernetes
         Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
-        Type=oneshot
 
-        StandardOutput=journal+console
+        ExecStart=/bin/sh -eu -c '\
-        StandardError=journal+console
+          echo "[ceph] waiting for API /readyz..." ; \
-        ExecStart=/bin/sh -c 'echo "witing 30s..."'
+          for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \
-        ExecStart=/bin/sleep 30s
+          echo "[ceph] apply manifests..." ; \
-        ExecStart=/bin/sh -c 'echo "create ceph namespace..."'
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml ; \
-        
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml ; \
-
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml ; \
-
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml ; \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml
+          echo "[ceph] done." \
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml
+        '
-        
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml
-        
-        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml
 
         ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done
-        ExecStart=/usr/bin/systemctl disable install-ceph.service
+        ExecStartPost=/usr/bin/systemctl start install-gitea.service
-        #RemainAfterExit=true
+
-        Restart=on-failure
-        RestartSec=120s
         [Install]
-        WantedBy=multi-user.target
+        WantedBy=undercloud-bootstrap.target
+
+    # --- Chain step 4 ---
     - name: install-gitea.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
-        Wants=install-ceph.service
+        Description=Install Gitea
+        Requires=install-ceph.service
         After=install-ceph.service
         ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done
 
         [Service]
+        Type=oneshot
         StandardOutput=journal+console
         StandardError=journal+console
         Environment=KUBECONFIG=/etc/kubernetes/admin.conf
         Environment=DATASTORE_TYPE=kubernetes
         Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
-        Type=oneshot
 
-        ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s
+        ExecStart=/bin/sh -eu -c '\
-        ExecStart=/bin/sleep 4m
+          echo "[gitea] wait for ceph provisioner..." ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml
+          kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml
+          echo "[gitea] apply manifests..." ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml ; \
-        ExecStart=/bin/sleep 60s
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml ; \
-        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml ; \
-        ExecStart=/bin/sleep 3m
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml ; \
-        ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh
+          echo "[gitea] wait for gitea deployment..." ; \
+          kubectl -n gitea wait deployment gitea --for=condition=Available=True --timeout=1200s ; \
+          echo "[gitea] run startup..." ; \
+          kubectl exec deploy/gitea -n gitea -- /bin/startup.sh ; \
+          echo "[gitea] done." \
+        '
 
         ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done
-        ExecStart=/usr/bin/systemctl disable install-gitea.service
+        ExecStartPost=/usr/bin/systemctl start install-argocd.service
-        Restart=on-failure
+
-        RestartSec=120s
         [Install]
-        WantedBy=multi-user.target
+        WantedBy=undercloud-bootstrap.target
+
+    # --- Chain step 5 ---
     - name: install-argocd.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
-        Wants=install-calico.service
+        Description=Install ArgoCD
-        After=install-calico.service
+        Requires=install-calico.service install-gitea.service
+        After=install-calico.service install-gitea.service
         ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done
+
         [Service]
+        Type=oneshot
         StandardOutput=journal+console
         StandardError=journal+console
-
         Environment=KUBECONFIG=/etc/kubernetes/admin.conf
         Environment=DATASTORE_TYPE=kubernetes
         Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
-        Type=oneshot
 
-        ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s
+        ExecStart=/bin/sh -eu -c '\
-        
+          echo "[argocd] wait for coredns..." ; \
-        ExecStart=/bin/sleep 1m
+          kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=1200s ; \
-        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml
+          echo "[argocd] install..." ; \
-        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml
+          kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml ; \
-        ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s
+          kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml ; \
-
+          kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=1200s ; \
-        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml
+          kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml ; \
-        ExecStart=/bin/sleep 10s
+          kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml ; \
-        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml
+          kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml ; \
-
+          echo "[argocd] done." \
-        ExecStart=/bin/sleep 10s
+        '
-        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml
-
-
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml
-        ##ExecStart=/bin/sleep 10m
-        #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s
-        #ExecStart=/bin/sleep 10m
-        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml
 
         ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done
-        ExecStart=/usr/bin/systemctl disable install-argocd.service
+        ExecStartPost=/usr/bin/systemctl start pin-service-ips.service
-        Restart=on-failure
+
-        RestartSec=120s
         [Install]
-        WantedBy=multi-user.target
+        WantedBy=undercloud-bootstrap.target
+
+    # --- Chain step 6 (final) ---
     - name: pin-service-ips.service
-      enabled: true
+      enabled: false
       contents: |
         [Unit]
         Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker
+        Requires=install-argocd.service install-calico.service kubeadm-init.service
         After=install-argocd.service install-calico.service kubeadm-init.service network-online.target
-        Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target
+        Wants=network-online.target
         ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done
 
-
         [Service]
         Type=oneshot
         StandardOutput=journal+console
@@ -607,8 +592,6 @@ systemd:
         '
 
         ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done
-        Restart=on-failure
-        RestartSec=120s
-        [Install]
-        WantedBy=multi-user.target
 
+        [Install]
+        WantedBy=undercloud-bootstrap.target

terraform/control-plane1.bu.old

@@ -0,0 +1,613 @@
+variant: flatcar
+version: 1.1.0
+
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud"
+
+storage:
+  directories:
+    - path: /opt/bin
+      overwrite: true
+      mode: 0755
+    - path: /opt/cni/bin
+      overwrite: true
+      mode: 755
+    - path: /etc/kubernetes/manifests
+      #overwrite: true
+      mode: 0755
+    - path: /etc/install-calico
+      overwrite: true
+      mode: 0755
+    - path: /var/lib/undercloud-stamps
+      mode: 0755
+
+  files:
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: |
+          control-plane1
+
+    - path: /etc/systemd/network/00-eth.network
+      mode: 0644
+      contents:
+        inline: |
+          [Match]
+          Name=eth*
+          
+          [Network]
+          Address=fd00:0:0:2::91/64
+          Address=2001:470:7116:2::91/64
+          Gateway=2001:470:7116:2::3
+          DNS=fd00:0:0:1::1
+          Address=10.0.2.91/24
+          Gateway=10.0.2.3
+          DNS=10.0.1.1
+          Domains=undercloud.local
+          IPv6AcceptRA=no
+          IPv6PrivacyExtensions=no
+
+    - path: /etc/hosts
+      mode: 0644
+      overwrite: true
+      contents:
+        inline: |
+          127.0.0.1   localhost
+          ::1         localhost ip6-localhost ip6-loopback
+
+          2001:470:7116:2::91  control-plane1.undercloud.local control-plane1
+          2001:470:7116:2::92  control-plane2.undercloud.local control-plane2
+          2001:470:7116:2::93  control-plane3.undercloud.local control-plane3
+          2001:470:7116:2::101 worker1.undercloud.local worker1
+          2001:470:7116:2::102 worker2.undercloud.local worker2
+          2001:470:7116:2::103 worker3.undercloud.local worker3
+
+          fd00:0:0:2::91      control-plane1.undercloud.local control-plane1
+          fd00:0:0:2::92      control-plane2.undercloud.local control-plane2
+          fd00:0:0:2::93      control-plane3.undercloud.local control-plane3
+          fd00:0:0:2::101     worker1.undercloud.local worker1
+          fd00:0:0:2::102     worker2.undercloud.local worker2
+          fd00:0:0:2::103     worker3.undercloud.local worker3
+
+          10.0.2.91   control-plane1.undercloud.local control-plane1
+          10.0.2.92   control-plane2.undercloud.local control-plane2
+          10.0.2.93   control-plane3.undercloud.local control-plane3
+          10.0.2.101  worker1.undercloud.local worker1
+          10.0.2.102  worker2.undercloud.local worker2
+          10.0.2.103  worker3.undercloud.local worker3
+
+    - path: /etc/motd
+      mode: 0644
+      overwrite: true
+      contents:
+        inline: |
+          *******************************************************************
+          *                        AUTHORIZED ACCESS ONLY                   *
+          *                                                                 *
+          *  This system is part of a secured infrastructure.               *
+          *  All activities are monitored and logged.                       *
+          *  Unauthorized access or misuse is strictly prohibited and       *
+          *  may result in disciplinary and legal action.                   *
+          *******************************************************************
+
+          --------------------------------------------------------------------------------
+          kubernetes controle plane Node
+
+          Manage via:
+          kubectl (kubectl)
+          calico (calicoctl)
+          velero - backup (velero)
+          argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/
+          --------------------------------------------------------------------------------
+
+    - path: /etc/sysctl.d/99-k8s.conf
+      mode: 0644
+      contents:
+        inline: |
+          net.ipv4.ip_forward                 = 1
+          net.ipv6.ip_forward                 = 1
+          net.ipv6.conf.all.forwarding        = 1
+          net.ipv4.conf.all.forwarding        = 1
+          net.bridge.bridge-nf-call-iptables  = 1
+          net.bridge.bridge-nf-call-ip6tables = 1
+          net.netfilter.nf_conntrack_max      = 1000000
+          net.ipv4.conf.all.rp_filter         = 0
+          net.ipv6.conf.all.disable_ipv6      = 0
+          vm.overcommit_memory                = 1
+          fs.inotify.max_user_watches         = 524288
+          fs.inotify.max_user_instances       = 4096
+          kernel.panic                        = 10
+          kernel.panic_on_oops                = 1
+
+    - path: /etc/flatcar/update.conf
+      overwrite: true
+      mode: 0420
+      contents:
+        inline: |
+          REBOOT_STRATEGY=off
+
+    - path: /opt/bin/kubeadm
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm"
+
+    - path: /opt/bin/kubelet
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet"
+
+    - path: /opt/bin/kubectl
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl"
+
+    - path: /opt/bin/calicoctl
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl"
+
+    - path: /opt/bin/velero
+      mode: 0755
+      contents:
+        source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero"
+
+    - path: /etc/kubernetes/kubeadm-init.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: InitConfiguration
+          bootstrapTokens:
+            - token: "kvg1hc.t3rewovrps426rof"
+              description: "default kubeadm bootstrap token"
+              ttl: "0"
+          nodeRegistration:
+            name: control-plane1
+            criSocket: unix:///run/containerd/containerd.sock
+            kubeletExtraArgs:
+              node-ip: "2001:470:7116:2::91"
+              cluster-dns: "10.0.91.53,2001:470:7116:f:1::53"
+              volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+          localAPIEndpoint:
+            advertiseAddress: "2001:470:7116:2::91"
+            bindPort: 6443
+          certificateKey: "fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b"
+          ---
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: ClusterConfiguration
+          controlPlaneEndpoint: "[fd00:0:0:2::100]:6443"
+          networking:
+            podSubnet: "2001:470:7116:a::/64,10.0.10.0/24"
+            serviceSubnet: "2001:470:7116:f:1::/108,10.0.91.0/24"
+            dnsDomain: "k8s.undercloud.local"
+          controllerManager:
+            extraArgs:
+              flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+              bind-address: '::'
+          apiServer:
+            extraArgs:
+              enable-aggregator-routing: "true"
+              proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt
+              proxy-client-key-file:  /etc/kubernetes/pki/front-proxy-client.key
+              requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt
+              requestheader-allowed-names: front-proxy-client
+              requestheader-extra-headers-prefix: X-Remote-Extra-
+              requestheader-group-headers:       X-Remote-Group
+              requestheader-username-headers:    X-Remote-User
+
+          ---
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          address: "::"
+          healthzBindAddress: "::"
+          clusterDomain: "k8s.undercloud.local"
+          clusterDNS:
+            - "2001:470:7116:f:1::53"
+            - "10.0.91.53"
+          volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec
+          cgroupDriver: "systemd"
+          authentication:
+            anonymous:
+              enabled: true
+            webhook:
+              enabled: true
+          authorization:
+            mode: Webhook
+
+    - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: kube-dns
+            namespace: kube-system
+            labels:
+              k8s-app: kube-dns
+          spec:
+            type: ClusterIP
+            ipFamilyPolicy: RequireDualStack
+            ipFamilies: [IPv6, IPv4]
+            clusterIP: 2001:470:7116:f:1::53
+            clusterIPs:
+              - 2001:470:7116:f:1::53
+              - 10.0.91.53
+            ports:
+              - name: dns
+                port: 53
+                protocol: UDP
+                targetPort: 53
+              - name: dns-tcp
+                port: 53
+                protocol: TCP
+                targetPort: 53
+              - name: metrics
+                port: 9153
+                protocol: TCP
+                targetPort: 9153
+            selector:
+              k8s-app: kube-dns
+
+
+systemd:
+  units:
+    - name: modules-load.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Load necessary kernel modules
+        Before=containerd.service kubeadm-init.service
+
+        [Service]
+        Type=oneshot
+
+        ExecStart=/usr/bin/modprobe br_netfilter
+        ExecStart=/usr/bin/modprobe overlay
+        RemainAfterExit=yes
+
+        [Install]
+        WantedBy=multi-user.target
+
+    - name: systemd-networkd-wait-online.service
+      enabled: true
+
+    - name: containerd.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=containerd container runtime
+        After=network.target modules-load.service
+
+        [Service]
+        #StandardOutput=journal+console
+        #StandardError=journal+console
+
+        ExecStart=/usr/bin/containerd
+        Restart=always
+        RestartSec=5
+        Delegate=yes
+        KillMode=process
+        OOMScoreAdjust=-999
+
+        [Install]
+        WantedBy=multi-user.target
+
+    - name: set-timezone.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Set Timezone
+        After=network-online.target
+        Wants=network-online.target
+        [Service]
+        StandardOutput=journal+console
+        StandardError=journal+console
+
+        ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"'
+        StandardOutput=journal+console
+        StandardError=journal+console
+        Type=oneshot
+        Restart=on-failure
+        ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin
+        ExecStart=/usr/bin/timedatectl set-ntp true 
+        [Install]
+        WantedBy=multi-user.target
+
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=kubelet, the Kubernetes Node Agent
+        Documentation=https://kubernets.io/docs/home
+        Wants=network-online.target
+        After=network-online.target
+        [Service]
+        #StandardOutput=journal+console
+        #StandardError=journal+console
+
+        #EnvironmentFile=/run/metadata/coreos
+        Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+        Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+        # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
+        EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+        ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+        Restart=always
+        StartLimitInterval=0
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+
+    - name: kubeadm-init.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubeadm Init Cluster
+        After=network-online.target containerd.service kubelet.service
+        Wants=network-online.target
+        ConditionPathExists=!/etc/kubernetes/kubelet.conf
+
+        [Service]
+        Type=oneshot
+        StandardOutput=journal+console
+        StandardError=journal+console
+
+        ExecStart=/bin/sh -c 'echo "kubeadm-init.service started..."'
+
+        # Environment
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=DATASTORE_TYPE=kubernetes
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/
+        
+        ExecStartPre=/bin/sleep 30s
+        ExecStart=/bin/sh -c 'echo "running kubeadm init..."'
+        ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml
+        
+        # copy files for kubectl
+        ExecStart=/bin/sh -c 'echo "copying files (admin.conf) to core home folder."'
+        ExecStartPost=/usr/bin/mkdir -p /home/core/.kube
+        ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config
+        ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config
+        
+        ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-calico.service
+      enabled: true
+      contents: |
+        [Unit]
+        Wants=kubeadm-init.service
+        After=kubeadm-init.service
+        ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done
+
+
+        [Service]
+        StandardOutput=journal+console
+        StandardError=journal+console
+
+        ExecStart=/bin/sh -c 'echo "install.calico.service started..."'
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=DATASTORE_TYPE=kubernetes
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
+        Type=oneshot
+        StandardOutput=journal+console
+        StandardError=journal+console
+        ExecStart=/bin/sh -c 'echo "witing 30s..."'
+        ExecStart=/bin/sleep 30s
+        ExecStart=/bin/sh -c 'echo "create calico namespace..."'
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml
+        ExecStart=/bin/sh -c 'echo "install tigera operator..."'
+        ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml
+        ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml
+        ExecStart=/bin/sh -c 'echo "witing 60s..."'
+        ExecStart=/bin/sleep 60s
+        ExecStart=/bin/sh -c 'echo "witing for tigera operator... (20mini max)"'
+        ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s
+        ExecStart=/bin/sh -c 'echo "create clico custom ressources..."'
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml
+        
+        ExecStart=/bin/sh -c 'echo "witing 3m.."'
+        ExecStart=/bin/sleep 3m
+        #ExecStart=/bin/sh -c 'echo "apply calico (calico-apiserver)..."'
+        #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml
+        #ExecStart=/bin/sh -c 'echo "witing 1m..."'
+        #ExecStart=/bin/sleep 2m
+        ExecStart=/bin/sh -c 'echo "witing calico-apiserver... (20mini max)"'
+        ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s
+        ExecStart=/bin/sh -c 'echo "witing 120s..."'
+        ExecStart=/bin/sleep 2m
+        ExecStart=/bin/sh -c 'echo "apply calico-peers..."'
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml
+        ExecStart=/bin/sh -c 'echo "witing 60s..."'
+        ExecStart=/bin/sleep 1m
+        ExecStart=/bin/sh -c 'echo "apply calico-ippools..."'
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml
+        
+        #ExecStart=/bin/sh -c 'echo "witing for whisker.."'
+        #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s
+        #ExecStart=/bin/sh -c 'echo "port-forward -n calico-system service/whisker 8081:8081"'
+        #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081
+        
+        ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done
+        ExecStart=/usr/bin/systemctl disable install-calico.service
+        #RemainAfterExit=true
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-ceph.service
+      enabled: true
+      contents: |
+        [Unit]
+        Wants=kubeadm-init.service
+        After=kubeadm-init.service
+        ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done
+
+        [Service]
+        StandardOutput=journal+console
+        StandardError=journal+console
+
+        ExecStart=/bin/sh -c 'echo "install.ceph.service started..."'
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=DATASTORE_TYPE=kubernetes
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
+        Type=oneshot
+        
+        StandardOutput=journal+console
+        StandardError=journal+console
+        ExecStart=/bin/sh -c 'echo "witing 30s..."'
+        ExecStart=/bin/sleep 30s
+        ExecStart=/bin/sh -c 'echo "create ceph namespace..."'
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml
+        
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml
+
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml
+
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml
+        
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml
+        
+        ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml
+        
+        ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done
+        ExecStart=/usr/bin/systemctl disable install-ceph.service
+        #RemainAfterExit=true
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-gitea.service
+      enabled: true
+      contents: |
+        [Unit]
+        Wants=install-ceph.service
+        After=install-ceph.service
+        ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done
+        
+        [Service]
+        StandardOutput=journal+console
+        StandardError=journal+console
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=DATASTORE_TYPE=kubernetes
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
+        Type=oneshot
+        
+        ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s
+        ExecStart=/bin/sleep 4m
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml
+        ExecStart=/bin/sleep 60s
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml
+        ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml
+        ExecStart=/bin/sleep 3m
+        ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh
+
+        ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done
+        ExecStart=/usr/bin/systemctl disable install-gitea.service
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-argocd.service
+      enabled: true
+      contents: |
+        [Unit]
+        Wants=install-calico.service
+        After=install-calico.service
+        ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done
+        [Service]
+        StandardOutput=journal+console
+        StandardError=journal+console
+
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=DATASTORE_TYPE=kubernetes
+        Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin
+        Type=oneshot
+
+        ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s
+        
+        ExecStart=/bin/sleep 1m
+        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml
+        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml
+        ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s
+
+        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml
+        ExecStart=/bin/sleep 10s
+        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml
+
+        ExecStart=/bin/sleep 10s
+        ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml
+
+
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml
+        ##ExecStart=/bin/sleep 10m
+        #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s
+        #ExecStart=/bin/sleep 10m
+        #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml
+        
+        ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done
+        ExecStart=/usr/bin/systemctl disable install-argocd.service
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+    - name: pin-service-ips.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker
+        After=install-argocd.service install-calico.service kubeadm-init.service network-online.target
+        Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target
+        ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done
+
+
+        [Service]
+        Type=oneshot
+        StandardOutput=journal+console
+        StandardError=journal+console
+        Environment=KUBECONFIG=/etc/kubernetes/admin.conf
+        Environment=PATH=/usr/bin:/usr/sbin:/opt/bin
+        ExecStart=/bin/sh -eu -c '\
+          echo "[pin-service-ips] waiting for API..." ; \
+          for i in $(seq 1 120); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \
+          echo "[pin-service-ips] ensure namespaces exist..." ; \
+          kubectl get ns kube-system >/dev/null ; \
+          kubectl get ns argocd >/dev/null 2>&1 || kubectl create ns argocd ; \
+          kubectl get ns calico-system >/dev/null ; \
+          echo "[pin-service-ips] wait for coredns/argocd readiness (best effort)..." ; \
+          kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \
+          kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \
+          echo "[pin-service-ips] replace Services with fixed ClusterIPs..." ; \
+          kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \
+          kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \
+          kubectl -n argocd delete svc argocd-server --ignore-not-found ; \
+          kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \
+          kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \
+          kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \
+          echo "[pin-service-ips] done." \
+        '
+
+        ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done
+        Restart=on-failure
+        RestartSec=120s
+        [Install]
+        WantedBy=multi-user.target
+

terraform/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.12.2",
-  "serial": 1227,
+  "serial": 1274,
   "lineage": "d92c42be-29f9-bad9-ef9a-3dc952ff5fa5",
   "outputs": {},
   "resources": [],