diff --git a/terraform/control-plane1.bu b/terraform/control-plane1.bu index ae2c492..9c42eeb 100644 --- a/terraform/control-plane1.bu +++ b/terraform/control-plane1.bu @@ -153,6 +153,11 @@ storage: mode: 0755 contents: source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero" + + - path: /opt/bin/remote-syslog + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/remote-syslog" - path: /etc/kubernetes/kubeadm-init.yaml mode: 0644 @@ -209,7 +214,6 @@ storage: - "10.0.91.53" volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec cgroupDriver: "systemd" - apiVersion: kubelet.config.k8s.io/v1beta1 authentication: anonymous: enabled: true @@ -253,9 +257,23 @@ storage: selector: k8s-app: kube-dns - systemd: units: + # --- Boot entrypoint: only this target is enabled at boot --- + - name: undercloud-bootstrap.target + enabled: true + contents: | + [Unit] + Description=Undercloud Bootstrap Chain + Wants=network-online.target + After=network-online.target + # Start the chain entry + Wants=containerd.service kubelet.service kubeadm-init.service + After=containerd.service kubelet.service kubeadm-init.service + + [Install] + WantedBy=multi-user.target + - name: modules-load.service enabled: true contents: | @@ -265,7 +283,6 @@ systemd: [Service] Type=oneshot - ExecStart=/usr/bin/modprobe br_netfilter ExecStart=/usr/bin/modprobe overlay RemainAfterExit=yes @@ -282,11 +299,9 @@ systemd: [Unit] Description=containerd container runtime After=network.target modules-load.service + Wants=modules-load.service [Service] - #StandardOutput=journal+console - #StandardError=journal+console - ExecStart=/usr/bin/containerd Restart=always RestartSec=5 @@ -304,17 +319,15 @@ systemd: Description=Set Timezone After=network-online.target Wants=network-online.target - [Service] - StandardOutput=journal+console - StandardError=journal+console - ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"' + [Service] + Type=oneshot StandardOutput=journal+console StandardError=journal+console - Type=oneshot - Restart=on-failure + ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"' ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin - ExecStart=/usr/bin/timedatectl set-ntp true + ExecStart=/usr/bin/timedatectl set-ntp true + [Install] WantedBy=multi-user.target @@ -323,32 +336,32 @@ systemd: contents: | [Unit] Description=kubelet, the Kubernetes Node Agent - Documentation=https://kubernets.io/docs/home + Documentation=https://kubernetes.io/docs/home Wants=network-online.target - After=network-online.target - [Service] - #StandardOutput=journal+console - #StandardError=journal+console + After=network-online.target containerd.service + Requires=containerd.service - #EnvironmentFile=/run/metadata/coreos + [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" - # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS Restart=always StartLimitInterval=0 RestartSec=10 + [Install] WantedBy=multi-user.target + # --- Chain step 1 --- - name: kubeadm-init.service - enabled: true + enabled: false contents: | [Unit] Description=Kubeadm Init Cluster - After=network-online.target containerd.service kubelet.service Wants=network-online.target + After=network-online.target containerd.service kubelet.service + Requires=containerd.service kubelet.service ConditionPathExists=!/etc/kubernetes/kubelet.conf [Service] @@ -356,230 +369,202 @@ systemd: StandardOutput=journal+console StandardError=journal+console - ExecStart=/bin/sh -c 'echo "kubeadm-init.service started..."' - - # Environment Environment=KUBECONFIG=/etc/kubernetes/admin.conf Environment=DATASTORE_TYPE=kubernetes Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/ - - ExecStartPre=/bin/sleep 30s - ExecStart=/bin/sh -c 'echo "running kubeadm init..."' - ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml - - # copy files for kubectl - ExecStart=/bin/sh -c 'echo "copying files (admin.conf) to core home folder."' - ExecStartPost=/usr/bin/mkdir -p /home/core/.kube - ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config - ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config - - ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service - Restart=on-failure - RestartSec=120s + + ExecStart=/bin/sh -eu -c '\ + echo "[kubeadm-init] started..." ; \ + echo "[kubeadm-init] waiting for containerd socket..." ; \ + for i in $(seq 1 60); do test -S /run/containerd/containerd.sock && break; sleep 1; done ; \ + echo "[kubeadm-init] running kubeadm init..." ; \ + /opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml ; \ + echo "[kubeadm-init] copying kubeconfig to core..." ; \ + mkdir -p /home/core/.kube ; \ + cp -f /etc/kubernetes/admin.conf /home/core/.kube/config ; \ + chown core:core /home/core/.kube/config ; \ + echo "[kubeadm-init] done." \ + ' + + # strictly start next step (serialization) + ExecStartPost=/usr/bin/systemctl start install-calico.service + [Install] - WantedBy=multi-user.target + WantedBy=undercloud-bootstrap.target + + # --- Chain step 2 --- - name: install-calico.service - enabled: true + enabled: false contents: | [Unit] - Wants=kubeadm-init.service + Description=Install Calico + Requires=kubeadm-init.service After=kubeadm-init.service ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done - [Service] - StandardOutput=journal+console - StandardError=journal+console - - ExecStart=/bin/sh -c 'echo "install.calico.service started..."' - Environment=KUBECONFIG=/etc/kubernetes/admin.conf - Environment=DATASTORE_TYPE=kubernetes - Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin Type=oneshot StandardOutput=journal+console StandardError=journal+console - ExecStart=/bin/sh -c 'echo "witing 30s..."' - ExecStart=/bin/sleep 30s - ExecStart=/bin/sh -c 'echo "create calico namespace..."' - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml - ExecStart=/bin/sh -c 'echo "install tigera operator..."' - ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml - ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml - ExecStart=/bin/sh -c 'echo "witing 60s..."' - ExecStart=/bin/sleep 60s - ExecStart=/bin/sh -c 'echo "witing for tigera operator... (20mini max)"' - ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s - ExecStart=/bin/sh -c 'echo "create clico custom ressources..."' - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml - - ExecStart=/bin/sh -c 'echo "witing 3m.."' - ExecStart=/bin/sleep 3m - #ExecStart=/bin/sh -c 'echo "apply calico (calico-apiserver)..."' - #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml - #ExecStart=/bin/sh -c 'echo "witing 1m..."' - #ExecStart=/bin/sleep 2m - ExecStart=/bin/sh -c 'echo "witing calico-apiserver... (20mini max)"' - ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s - ExecStart=/bin/sh -c 'echo "witing 120s..."' - ExecStart=/bin/sleep 2m - ExecStart=/bin/sh -c 'echo "apply calico-peers..."' - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml - ExecStart=/bin/sh -c 'echo "witing 60s..."' - ExecStart=/bin/sleep 1m - ExecStart=/bin/sh -c 'echo "apply calico-ippools..."' - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml - - #ExecStart=/bin/sh -c 'echo "witing for whisker.."' - #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s - #ExecStart=/bin/sh -c 'echo "port-forward -n calico-system service/whisker 8081:8081"' - #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081 - + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin + + ExecStart=/bin/sh -eu -c '\ + echo "[calico] waiting for API /readyz..." ; \ + for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ + echo "[calico] create namespace + operator..." ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml ; \ + kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml || true ; \ + kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml || true ; \ + echo "[calico] wait for tigera-operator..." ; \ + kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s ; \ + echo "[calico] apply custom resources..." ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml ; \ + echo "[calico] wait for calico-apiserver..." ; \ + kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s ; \ + echo "[calico] apply peers + pools..." ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml ; \ + echo "[calico] done." \ + ' + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done - ExecStart=/usr/bin/systemctl disable install-calico.service - #RemainAfterExit=true - Restart=on-failure - RestartSec=120s + ExecStartPost=/usr/bin/systemctl start install-ceph.service + [Install] - WantedBy=multi-user.target + WantedBy=undercloud-bootstrap.target + + # --- Chain step 3 --- - name: install-ceph.service - enabled: true + enabled: false contents: | [Unit] - Wants=kubeadm-init.service - After=kubeadm-init.service + Description=Install Ceph CSI + Requires=install-calico.service + After=install-calico.service ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done [Service] + Type=oneshot StandardOutput=journal+console StandardError=journal+console - - ExecStart=/bin/sh -c 'echo "install.ceph.service started..."' Environment=KUBECONFIG=/etc/kubernetes/admin.conf Environment=DATASTORE_TYPE=kubernetes Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - Type=oneshot - - StandardOutput=journal+console - StandardError=journal+console - ExecStart=/bin/sh -c 'echo "witing 30s..."' - ExecStart=/bin/sleep 30s - ExecStart=/bin/sh -c 'echo "create ceph namespace..."' - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml - - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml + ExecStart=/bin/sh -eu -c '\ + echo "[ceph] waiting for API /readyz..." ; \ + for i in $(seq 1 180); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ + echo "[ceph] apply manifests..." ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml ; \ + echo "[ceph] done." \ + ' - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml - - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml - - ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done - ExecStart=/usr/bin/systemctl disable install-ceph.service - #RemainAfterExit=true - Restart=on-failure - RestartSec=120s + ExecStartPost=/usr/bin/systemctl start install-gitea.service + [Install] - WantedBy=multi-user.target + WantedBy=undercloud-bootstrap.target + + # --- Chain step 4 --- - name: install-gitea.service - enabled: true + enabled: false contents: | [Unit] - Wants=install-ceph.service + Description=Install Gitea + Requires=install-ceph.service After=install-ceph.service ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done - + [Service] + Type=oneshot StandardOutput=journal+console StandardError=journal+console Environment=KUBECONFIG=/etc/kubernetes/admin.conf Environment=DATASTORE_TYPE=kubernetes Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - Type=oneshot - - ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s - ExecStart=/bin/sleep 4m - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml - ExecStart=/bin/sleep 60s - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml - ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml - ExecStart=/bin/sleep 3m - ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh + + ExecStart=/bin/sh -eu -c '\ + echo "[gitea] wait for ceph provisioner..." ; \ + kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s ; \ + echo "[gitea] apply manifests..." ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml ; \ + echo "[gitea] wait for gitea deployment..." ; \ + kubectl -n gitea wait deployment gitea --for=condition=Available=True --timeout=1200s ; \ + echo "[gitea] run startup..." ; \ + kubectl exec deploy/gitea -n gitea -- /bin/startup.sh ; \ + echo "[gitea] done." \ + ' ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done - ExecStart=/usr/bin/systemctl disable install-gitea.service - Restart=on-failure - RestartSec=120s + ExecStartPost=/usr/bin/systemctl start install-argocd.service + [Install] - WantedBy=multi-user.target + WantedBy=undercloud-bootstrap.target + + # --- Chain step 5 --- - name: install-argocd.service - enabled: true + enabled: false contents: | [Unit] - Wants=install-calico.service - After=install-calico.service + Description=Install ArgoCD + Requires=install-calico.service install-gitea.service + After=install-calico.service install-gitea.service ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done + [Service] + Type=oneshot StandardOutput=journal+console StandardError=journal+console - Environment=KUBECONFIG=/etc/kubernetes/admin.conf Environment=DATASTORE_TYPE=kubernetes Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin - Type=oneshot - ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s - - ExecStart=/bin/sleep 1m - ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml - ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml - ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s + ExecStart=/bin/sh -eu -c '\ + echo "[argocd] wait for coredns..." ; \ + kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=1200s ; \ + echo "[argocd] install..." ; \ + kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml ; \ + kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml ; \ + kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=1200s ; \ + kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml ; \ + kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml ; \ + kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml ; \ + echo "[argocd] done." \ + ' - ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml - ExecStart=/bin/sleep 10s - ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml - - ExecStart=/bin/sleep 10s - ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml - - - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml - ##ExecStart=/bin/sleep 10m - #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s - #ExecStart=/bin/sleep 10m - #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml - ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done - ExecStart=/usr/bin/systemctl disable install-argocd.service - Restart=on-failure - RestartSec=120s + ExecStartPost=/usr/bin/systemctl start pin-service-ips.service + [Install] - WantedBy=multi-user.target + WantedBy=undercloud-bootstrap.target + + # --- Chain step 6 (final) --- - name: pin-service-ips.service - enabled: true + enabled: false contents: | [Unit] Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker + Requires=install-argocd.service install-calico.service kubeadm-init.service After=install-argocd.service install-calico.service kubeadm-init.service network-online.target - Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target + Wants=network-online.target ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done - [Service] Type=oneshot StandardOutput=journal+console @@ -607,8 +592,6 @@ systemd: ' ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done - Restart=on-failure - RestartSec=120s - [Install] - WantedBy=multi-user.target + [Install] + WantedBy=undercloud-bootstrap.target diff --git a/terraform/control-plane1.bu.old b/terraform/control-plane1.bu.old new file mode 100644 index 0000000..81e3ddd --- /dev/null +++ b/terraform/control-plane1.bu.old @@ -0,0 +1,613 @@ +variant: flatcar +version: 1.1.0 + +passwd: + users: + - name: core + ssh_authorized_keys: + - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud" + +storage: + directories: + - path: /opt/bin + overwrite: true + mode: 0755 + - path: /opt/cni/bin + overwrite: true + mode: 755 + - path: /etc/kubernetes/manifests + #overwrite: true + mode: 0755 + - path: /etc/install-calico + overwrite: true + mode: 0755 + - path: /var/lib/undercloud-stamps + mode: 0755 + + files: + - path: /etc/hostname + mode: 0644 + contents: + inline: | + control-plane1 + + - path: /etc/systemd/network/00-eth.network + mode: 0644 + contents: + inline: | + [Match] + Name=eth* + + [Network] + Address=fd00:0:0:2::91/64 + Address=2001:470:7116:2::91/64 + Gateway=2001:470:7116:2::3 + DNS=fd00:0:0:1::1 + Address=10.0.2.91/24 + Gateway=10.0.2.3 + DNS=10.0.1.1 + Domains=undercloud.local + IPv6AcceptRA=no + IPv6PrivacyExtensions=no + + - path: /etc/hosts + mode: 0644 + overwrite: true + contents: + inline: | + 127.0.0.1 localhost + ::1 localhost ip6-localhost ip6-loopback + + 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1 + 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2 + 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3 + 2001:470:7116:2::101 worker1.undercloud.local worker1 + 2001:470:7116:2::102 worker2.undercloud.local worker2 + 2001:470:7116:2::103 worker3.undercloud.local worker3 + + fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 + fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 + fd00:0:0:2::93 control-plane3.undercloud.local control-plane3 + fd00:0:0:2::101 worker1.undercloud.local worker1 + fd00:0:0:2::102 worker2.undercloud.local worker2 + fd00:0:0:2::103 worker3.undercloud.local worker3 + + 10.0.2.91 control-plane1.undercloud.local control-plane1 + 10.0.2.92 control-plane2.undercloud.local control-plane2 + 10.0.2.93 control-plane3.undercloud.local control-plane3 + 10.0.2.101 worker1.undercloud.local worker1 + 10.0.2.102 worker2.undercloud.local worker2 + 10.0.2.103 worker3.undercloud.local worker3 + + - path: /etc/motd + mode: 0644 + overwrite: true + contents: + inline: | + ******************************************************************* + * AUTHORIZED ACCESS ONLY * + * * + * This system is part of a secured infrastructure. * + * All activities are monitored and logged. * + * Unauthorized access or misuse is strictly prohibited and * + * may result in disciplinary and legal action. * + ******************************************************************* + + -------------------------------------------------------------------------------- + kubernetes controle plane Node + + Manage via: + kubectl (kubectl) + calico (calicoctl) + velero - backup (velero) + argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/ + -------------------------------------------------------------------------------- + + - path: /etc/sysctl.d/99-k8s.conf + mode: 0644 + contents: + inline: | + net.ipv4.ip_forward = 1 + net.ipv6.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + net.ipv4.conf.all.forwarding = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.netfilter.nf_conntrack_max = 1000000 + net.ipv4.conf.all.rp_filter = 0 + net.ipv6.conf.all.disable_ipv6 = 0 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 524288 + fs.inotify.max_user_instances = 4096 + kernel.panic = 10 + kernel.panic_on_oops = 1 + + - path: /etc/flatcar/update.conf + overwrite: true + mode: 0420 + contents: + inline: | + REBOOT_STRATEGY=off + + - path: /opt/bin/kubeadm + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm" + + - path: /opt/bin/kubelet + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet" + + - path: /opt/bin/kubectl + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl" + + - path: /opt/bin/calicoctl + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl" + + - path: /opt/bin/velero + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero" + + - path: /etc/kubernetes/kubeadm-init.yaml + mode: 0644 + contents: + inline: | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: InitConfiguration + bootstrapTokens: + - token: "kvg1hc.t3rewovrps426rof" + description: "default kubeadm bootstrap token" + ttl: "0" + nodeRegistration: + name: control-plane1 + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + node-ip: "2001:470:7116:2::91" + cluster-dns: "10.0.91.53,2001:470:7116:f:1::53" + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + localAPIEndpoint: + advertiseAddress: "2001:470:7116:2::91" + bindPort: 6443 + certificateKey: "fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b" + --- + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + controlPlaneEndpoint: "[fd00:0:0:2::100]:6443" + networking: + podSubnet: "2001:470:7116:a::/64,10.0.10.0/24" + serviceSubnet: "2001:470:7116:f:1::/108,10.0.91.0/24" + dnsDomain: "k8s.undercloud.local" + controllerManager: + extraArgs: + flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + bind-address: '::' + apiServer: + extraArgs: + enable-aggregator-routing: "true" + proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt + proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key + requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt + requestheader-allowed-names: front-proxy-client + requestheader-extra-headers-prefix: X-Remote-Extra- + requestheader-group-headers: X-Remote-Group + requestheader-username-headers: X-Remote-User + + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + address: "::" + healthzBindAddress: "::" + clusterDomain: "k8s.undercloud.local" + clusterDNS: + - "2001:470:7116:f:1::53" + - "10.0.91.53" + volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec + cgroupDriver: "systemd" + authentication: + anonymous: + enabled: true + webhook: + enabled: true + authorization: + mode: Webhook + + - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + spec: + type: ClusterIP + ipFamilyPolicy: RequireDualStack + ipFamilies: [IPv6, IPv4] + clusterIP: 2001:470:7116:f:1::53 + clusterIPs: + - 2001:470:7116:f:1::53 + - 10.0.91.53 + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + - name: metrics + port: 9153 + protocol: TCP + targetPort: 9153 + selector: + k8s-app: kube-dns + + +systemd: + units: + - name: modules-load.service + enabled: true + contents: | + [Unit] + Description=Load necessary kernel modules + Before=containerd.service kubeadm-init.service + + [Service] + Type=oneshot + + ExecStart=/usr/bin/modprobe br_netfilter + ExecStart=/usr/bin/modprobe overlay + RemainAfterExit=yes + + [Install] + WantedBy=multi-user.target + + - name: systemd-networkd-wait-online.service + enabled: true + + - name: containerd.service + enabled: true + contents: | + [Unit] + Description=containerd container runtime + After=network.target modules-load.service + + [Service] + #StandardOutput=journal+console + #StandardError=journal+console + + ExecStart=/usr/bin/containerd + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + OOMScoreAdjust=-999 + + [Install] + WantedBy=multi-user.target + + - name: set-timezone.service + enabled: true + contents: | + [Unit] + Description=Set Timezone + After=network-online.target + Wants=network-online.target + [Service] + StandardOutput=journal+console + StandardError=journal+console + + ExecStart=/bin/sh -c 'echo "setting timezone to Europe/Berlin"' + StandardOutput=journal+console + StandardError=journal+console + Type=oneshot + Restart=on-failure + ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin + ExecStart=/usr/bin/timedatectl set-ntp true + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=kubelet, the Kubernetes Node Agent + Documentation=https://kubernets.io/docs/home + Wants=network-online.target + After=network-online.target + [Service] + #StandardOutput=journal+console + #StandardError=journal+console + + #EnvironmentFile=/run/metadata/coreos + Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" + Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" + # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically + EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env + ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + Restart=always + StartLimitInterval=0 + RestartSec=10 + [Install] + WantedBy=multi-user.target + + - name: kubeadm-init.service + enabled: true + contents: | + [Unit] + Description=Kubeadm Init Cluster + After=network-online.target containerd.service kubelet.service + Wants=network-online.target + ConditionPathExists=!/etc/kubernetes/kubelet.conf + + [Service] + Type=oneshot + StandardOutput=journal+console + StandardError=journal+console + + ExecStart=/bin/sh -c 'echo "kubeadm-init.service started..."' + + # Environment + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/ + + ExecStartPre=/bin/sleep 30s + ExecStart=/bin/sh -c 'echo "running kubeadm init..."' + ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml + + # copy files for kubectl + ExecStart=/bin/sh -c 'echo "copying files (admin.conf) to core home folder."' + ExecStartPost=/usr/bin/mkdir -p /home/core/.kube + ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config + ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config + + ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + - name: install-calico.service + enabled: true + contents: | + [Unit] + Wants=kubeadm-init.service + After=kubeadm-init.service + ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done + + + [Service] + StandardOutput=journal+console + StandardError=journal+console + + ExecStart=/bin/sh -c 'echo "install.calico.service started..."' + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin + Type=oneshot + StandardOutput=journal+console + StandardError=journal+console + ExecStart=/bin/sh -c 'echo "witing 30s..."' + ExecStart=/bin/sleep 30s + ExecStart=/bin/sh -c 'echo "create calico namespace..."' + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml + ExecStart=/bin/sh -c 'echo "install tigera operator..."' + ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml + ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml + ExecStart=/bin/sh -c 'echo "witing 60s..."' + ExecStart=/bin/sleep 60s + ExecStart=/bin/sh -c 'echo "witing for tigera operator... (20mini max)"' + ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s + ExecStart=/bin/sh -c 'echo "create clico custom ressources..."' + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml + + ExecStart=/bin/sh -c 'echo "witing 3m.."' + ExecStart=/bin/sleep 3m + #ExecStart=/bin/sh -c 'echo "apply calico (calico-apiserver)..."' + #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml + #ExecStart=/bin/sh -c 'echo "witing 1m..."' + #ExecStart=/bin/sleep 2m + ExecStart=/bin/sh -c 'echo "witing calico-apiserver... (20mini max)"' + ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s + ExecStart=/bin/sh -c 'echo "witing 120s..."' + ExecStart=/bin/sleep 2m + ExecStart=/bin/sh -c 'echo "apply calico-peers..."' + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml + ExecStart=/bin/sh -c 'echo "witing 60s..."' + ExecStart=/bin/sleep 1m + ExecStart=/bin/sh -c 'echo "apply calico-ippools..."' + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml + + #ExecStart=/bin/sh -c 'echo "witing for whisker.."' + #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s + #ExecStart=/bin/sh -c 'echo "port-forward -n calico-system service/whisker 8081:8081"' + #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081 + + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done + ExecStart=/usr/bin/systemctl disable install-calico.service + #RemainAfterExit=true + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + - name: install-ceph.service + enabled: true + contents: | + [Unit] + Wants=kubeadm-init.service + After=kubeadm-init.service + ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done + + [Service] + StandardOutput=journal+console + StandardError=journal+console + + ExecStart=/bin/sh -c 'echo "install.ceph.service started..."' + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin + Type=oneshot + + StandardOutput=journal+console + StandardError=journal+console + ExecStart=/bin/sh -c 'echo "witing 30s..."' + ExecStart=/bin/sleep 30s + ExecStart=/bin/sh -c 'echo "create ceph namespace..."' + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml + + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml + + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml + + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml + + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml + + ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml + + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done + ExecStart=/usr/bin/systemctl disable install-ceph.service + #RemainAfterExit=true + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + - name: install-gitea.service + enabled: true + contents: | + [Unit] + Wants=install-ceph.service + After=install-ceph.service + ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done + + [Service] + StandardOutput=journal+console + StandardError=journal+console + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin + Type=oneshot + + ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s + ExecStart=/bin/sleep 4m + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml + ExecStart=/bin/sleep 60s + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml + ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml + ExecStart=/bin/sleep 3m + ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh + + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done + ExecStart=/usr/bin/systemctl disable install-gitea.service + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + - name: install-argocd.service + enabled: true + contents: | + [Unit] + Wants=install-calico.service + After=install-calico.service + ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done + [Service] + StandardOutput=journal+console + StandardError=journal+console + + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin + Type=oneshot + + ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s + + ExecStart=/bin/sleep 1m + ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml + ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml + ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s + + ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml + ExecStart=/bin/sleep 10s + ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml + + ExecStart=/bin/sleep 10s + ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml + + + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml + ##ExecStart=/bin/sleep 10m + #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s + #ExecStart=/bin/sleep 10m + #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml + + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done + ExecStart=/usr/bin/systemctl disable install-argocd.service + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + - name: pin-service-ips.service + enabled: true + contents: | + [Unit] + Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker + After=install-argocd.service install-calico.service kubeadm-init.service network-online.target + Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target + ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done + + + [Service] + Type=oneshot + StandardOutput=journal+console + StandardError=journal+console + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=PATH=/usr/bin:/usr/sbin:/opt/bin + ExecStart=/bin/sh -eu -c '\ + echo "[pin-service-ips] waiting for API..." ; \ + for i in $(seq 1 120); do kubectl get --raw=/readyz >/dev/null 2>&1 && break; sleep 2; done ; \ + echo "[pin-service-ips] ensure namespaces exist..." ; \ + kubectl get ns kube-system >/dev/null ; \ + kubectl get ns argocd >/dev/null 2>&1 || kubectl create ns argocd ; \ + kubectl get ns calico-system >/dev/null ; \ + echo "[pin-service-ips] wait for coredns/argocd readiness (best effort)..." ; \ + kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \ + kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \ + echo "[pin-service-ips] replace Services with fixed ClusterIPs..." ; \ + kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \ + kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \ + kubectl -n argocd delete svc argocd-server --ignore-not-found ; \ + kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \ + kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \ + kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \ + echo "[pin-service-ips] done." \ + ' + + ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done + Restart=on-failure + RestartSec=120s + [Install] + WantedBy=multi-user.target + diff --git a/terraform/terraform.tfstate b/terraform/terraform.tfstate index e076732..6715e69 100644 --- a/terraform/terraform.tfstate +++ b/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.12.2", - "serial": 1247, + "serial": 1294, "lineage": "d92c42be-29f9-bad9-ef9a-3dc952ff5fa5", "outputs": {}, "resources": [ @@ -14,10 +14,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 755\n - path: /etc/kubernetes/manifests\n #overwrite: true\n mode: 0755\n - path: /etc/install-calico\n overwrite: true\n mode: 0755\n - path: /var/lib/undercloud-stamps\n mode: 0755\n\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::91/64\n Address=2001:470:7116:2::91/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.91/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /opt/bin/velero\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\"\n\n - path: /etc/kubernetes/kubeadm-init.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n bootstrapTokens:\n - token: \"kvg1hc.t3rewovrps426rof\"\n description: \"default kubeadm bootstrap token\"\n ttl: \"0\"\n nodeRegistration:\n name: control-plane1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::91\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::91\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n controlPlaneEndpoint: \"[fd00:0:0:2::100]:6443\"\n networking:\n podSubnet: \"2001:470:7116:a::/64,10.0.10.0/24\"\n serviceSubnet: \"2001:470:7116:f:1::/108,10.0.91.0/24\"\n dnsDomain: \"k8s.undercloud.local\"\n controllerManager:\n extraArgs:\n flex-volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n bind-address: '::'\n apiServer:\n extraArgs:\n enable-aggregator-routing: \"true\"\n proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt\n proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key\n requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt\n requestheader-allowed-names: front-proxy-client\n requestheader-extra-headers-prefix: X-Remote-Extra-\n requestheader-group-headers: X-Remote-Group\n requestheader-username-headers: X-Remote-User\n\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec\n cgroupDriver: \"systemd\"\n apiVersion: kubelet.config.k8s.io/v1beta1\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\n - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: v1\n kind: Service\n metadata:\n name: kube-dns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n spec:\n type: ClusterIP\n ipFamilyPolicy: RequireDualStack\n ipFamilies: [IPv6, IPv4]\n clusterIP: 2001:470:7116:f:1::53\n clusterIPs:\n - 2001:470:7116:f:1::53\n - 10.0.91.53\n ports:\n - name: dns\n port: 53\n protocol: UDP\n targetPort: 53\n - name: dns-tcp\n port: 53\n protocol: TCP\n targetPort: 53\n - name: metrics\n port: 9153\n protocol: TCP\n targetPort: 9153\n selector:\n k8s-app: kube-dns\n\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"setting timezone to Europe/Berlin\"'\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-init.service\n enabled: true\n contents: |\n [Unit]\n Description=Kubeadm Init Cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n ConditionPathExists=!/etc/kubernetes/kubelet.conf\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"kubeadm-init.service started...\"'\n\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"running kubeadm init...\"'\n ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\n \n # copy files for kubectl\n ExecStart=/bin/sh -c 'echo \"copying files (admin.conf) to core home folder.\"'\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-calico.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\n\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.calico.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create calico namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\n ExecStart=/bin/sh -c 'echo \"install tigera operator...\"'\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 60s\n ExecStart=/bin/sh -c 'echo \"witing for tigera operator... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"create clico custom ressources...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\n \n ExecStart=/bin/sh -c 'echo \"witing 3m..\"'\n ExecStart=/bin/sleep 3m\n #ExecStart=/bin/sh -c 'echo \"apply calico (calico-apiserver)...\"'\n #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\n #ExecStart=/bin/sh -c 'echo \"witing 1m...\"'\n #ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"witing calico-apiserver... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"witing 120s...\"'\n ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"apply calico-peers...\"'\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 1m\n ExecStart=/bin/sh -c 'echo \"apply calico-ippools...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\n \n #ExecStart=/bin/sh -c 'echo \"witing for whisker..\"'\n #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\n #ExecStart=/bin/sh -c 'echo \"port-forward -n calico-system service/whisker 8081:8081\"'\n #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\n ExecStart=/usr/bin/systemctl disable install-calico.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-ceph.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.ceph.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create ceph namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\n ExecStart=/usr/bin/systemctl disable install-ceph.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-gitea.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-ceph.service\n After=install-ceph.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\n \n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sleep 4m\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\n ExecStart=/bin/sleep 60s\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\n ExecStart=/bin/sleep 3m\n ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\n ExecStart=/usr/bin/systemctl disable install-gitea.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-argocd.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n\n ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\n \n ExecStart=/bin/sleep 1m\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\n ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\n\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\n\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\n\n\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\n ##ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\n #ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\n ExecStart=/usr/bin/systemctl disable install-argocd.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: pin-service-ips.service\n enabled: true\n contents: |\n [Unit]\n Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\n After=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\n\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n ExecStart=/bin/sh -eu -c '\\\n echo \"[pin-service-ips] waiting for API...\" ; \\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \u003e/dev/null 2\u003e\u00261 \u0026\u0026 break; sleep 2; done ; \\\n echo \"[pin-service-ips] ensure namespaces exist...\" ; \\\n kubectl get ns kube-system \u003e/dev/null ; \\\n kubectl get ns argocd \u003e/dev/null 2\u003e\u00261 || kubectl create ns argocd ; \\\n kubectl get ns calico-system \u003e/dev/null ; \\\n echo \"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\" ; \\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\n echo \"[pin-service-ips] replace Services with fixed ClusterIPs...\" ; \\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\n echo \"[pin-service-ips] done.\" \\\n '\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n", - "id": "2254240921", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 755\n - path: /etc/kubernetes/manifests\n #overwrite: true\n mode: 0755\n - path: /etc/install-calico\n overwrite: true\n mode: 0755\n - path: /var/lib/undercloud-stamps\n mode: 0755\n\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::91/64\n Address=2001:470:7116:2::91/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.91/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /opt/bin/velero\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\"\n\n - path: /etc/kubernetes/kubeadm-init.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n bootstrapTokens:\n - token: \"kvg1hc.t3rewovrps426rof\"\n description: \"default kubeadm bootstrap token\"\n ttl: \"0\"\n nodeRegistration:\n name: control-plane1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::91\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::91\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n controlPlaneEndpoint: \"[fd00:0:0:2::100]:6443\"\n networking:\n podSubnet: \"2001:470:7116:a::/64,10.0.10.0/24\"\n serviceSubnet: \"2001:470:7116:f:1::/108,10.0.91.0/24\"\n dnsDomain: \"k8s.undercloud.local\"\n controllerManager:\n extraArgs:\n flex-volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n bind-address: '::'\n apiServer:\n extraArgs:\n enable-aggregator-routing: \"true\"\n proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt\n proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key\n requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt\n requestheader-allowed-names: front-proxy-client\n requestheader-extra-headers-prefix: X-Remote-Extra-\n requestheader-group-headers: X-Remote-Group\n requestheader-username-headers: X-Remote-User\n\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec\n cgroupDriver: \"systemd\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\n - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: v1\n kind: Service\n metadata:\n name: kube-dns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n spec:\n type: ClusterIP\n ipFamilyPolicy: RequireDualStack\n ipFamilies: [IPv6, IPv4]\n clusterIP: 2001:470:7116:f:1::53\n clusterIPs:\n - 2001:470:7116:f:1::53\n - 10.0.91.53\n ports:\n - name: dns\n port: 53\n protocol: UDP\n targetPort: 53\n - name: dns-tcp\n port: 53\n protocol: TCP\n targetPort: 53\n - name: metrics\n port: 9153\n protocol: TCP\n targetPort: 9153\n selector:\n k8s-app: kube-dns\n\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"setting timezone to Europe/Berlin\"'\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-init.service\n enabled: true\n contents: |\n [Unit]\n Description=Kubeadm Init Cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n ConditionPathExists=!/etc/kubernetes/kubelet.conf\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"kubeadm-init.service started...\"'\n\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"running kubeadm init...\"'\n ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\n \n # copy files for kubectl\n ExecStart=/bin/sh -c 'echo \"copying files (admin.conf) to core home folder.\"'\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-calico.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\n\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.calico.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create calico namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\n ExecStart=/bin/sh -c 'echo \"install tigera operator...\"'\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 60s\n ExecStart=/bin/sh -c 'echo \"witing for tigera operator... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"create clico custom ressources...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\n \n ExecStart=/bin/sh -c 'echo \"witing 3m..\"'\n ExecStart=/bin/sleep 3m\n #ExecStart=/bin/sh -c 'echo \"apply calico (calico-apiserver)...\"'\n #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\n #ExecStart=/bin/sh -c 'echo \"witing 1m...\"'\n #ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"witing calico-apiserver... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"witing 120s...\"'\n ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"apply calico-peers...\"'\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 1m\n ExecStart=/bin/sh -c 'echo \"apply calico-ippools...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\n \n #ExecStart=/bin/sh -c 'echo \"witing for whisker..\"'\n #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\n #ExecStart=/bin/sh -c 'echo \"port-forward -n calico-system service/whisker 8081:8081\"'\n #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\n ExecStart=/usr/bin/systemctl disable install-calico.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-ceph.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.ceph.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create ceph namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\n ExecStart=/usr/bin/systemctl disable install-ceph.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-gitea.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-ceph.service\n After=install-ceph.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\n \n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sleep 4m\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\n ExecStart=/bin/sleep 60s\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\n ExecStart=/bin/sleep 3m\n ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\n ExecStart=/usr/bin/systemctl disable install-gitea.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-argocd.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n\n ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\n \n ExecStart=/bin/sleep 1m\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\n ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\n\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\n\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\n\n\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\n ##ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\n #ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\n ExecStart=/usr/bin/systemctl disable install-argocd.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: pin-service-ips.service\n enabled: true\n contents: |\n [Unit]\n Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\n After=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\n\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n ExecStart=/bin/sh -eu -c '\\\n echo \"[pin-service-ips] waiting for API...\" ; \\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \u003e/dev/null 2\u003e\u00261 \u0026\u0026 break; sleep 2; done ; \\\n echo \"[pin-service-ips] ensure namespaces exist...\" ; \\\n kubectl get ns kube-system \u003e/dev/null ; \\\n kubectl get ns argocd \u003e/dev/null 2\u003e\u00261 || kubectl create ns argocd ; \\\n kubectl get ns calico-system \u003e/dev/null ; \\\n echo \"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\" ; \\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\n echo \"[pin-service-ips] replace Services with fixed ClusterIPs...\" ; \\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\n echo \"[pin-service-ips] done.\" \\\n '\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n", + "id": "1444300035", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xVTW/jNhC981cQvOxlqQ9LsRPe0s2iWCxaBJt+AUUPFDmSCdGkOqQce399Qck2oq6DpkVtwJCheU+PM++N5GB+AQzGO0H7sQGpd1l/GzLj833ZQJQV6Y3Tgn5yJn7wrjXdiDIa70jjfQwR5fCT78EFQSjlNKZrQVm/78qtymKF8Oz3OIR6tUbfMkIppRqCQjPE6alMQytHG8+PpxfemWyGxGgFZQUjzmv4Ap1JFRMBodTJHQiqvIvoLR+sdFASShWaJ696iIKOzhxEnuc4ujzVSeMA9YvLLHjVEzqJsBA/HiLKe+ymU1GaHsrNIChbFUUp6k0hNmW5Fish7spZoLJjiIBcuyAoK4usyO7K7KZ6v0S0ohTippoxe2/HHfDBjp1xXBsUlOV+iLk1DRxA5UkNOogQ8pOwU3HIZ2w+lTFivZL2/vHTR6cHb1xMsqXeA0YT4F5rhBBeV98Ypx89RkHXdV0RlWCtUTLCZzgKylqAjargRrVKqg3Um02t1tAqWdTrWtYrfXu3aaui3VRFUdWFXt9sbpsb1Ta1rHRVN4xwzsmbrfZhbuXSbafpPqbhXk5J2e+tLgqRvishyqL4Q6QjMOIgPnvsjetSKwavn8bGJSf8rQdSiHxdv5/mlX7yVZ06EgD3RsEroGmIeVncvj/P+YzTLjz4nTRTBG5DNjoNqKwfdTaNiJ3PYQF/kE52gEkfLO3WWjjw/88ciTKNmMuzEd4J8S7N4wlwf1UBONlY4LLrEDoZPXL0Y0zdpCziCDPpgP5w5MoacJEn1/DWWBA0h7iQN/Qmb9G7yF8iMoXxW5oejieWt9L0cJxoEP4cIcQtSA14USXfqkle9CyJpLX+GTRPSyYI+q2AK5ipm3z+E/iA0JqDoL/xL7DzEfi0XfgVXId+HM44QefPBfZ9unsFNQbApG4BvKB+DoDkagAtxExNIVvmsDzl8PNcs8zhxUNMCEa2IG3cfv3OOH2/uHFah/8UhlPVj0/z24O9uiz5y53KyGzwx8nuDyka/yEZRE3tfkCTMkBZOIYIO83+RZ/kGLfgYlqVp1eRdN4dd35c5EgLmkJDKH2GZut9f+1m4vJovl6odl6DoL/OCPJXAAAA///31/bnqQcAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -161,7 +161,7 @@ { "schema_version": 0, "attributes": { - "id": "6225671499492048727", + "id": "6387838205690992951", "triggers": null }, "sensitive_attributes": [], @@ -193,7 +193,7 @@ { "schema_version": 0, "attributes": { - "id": "7412815847183625707", + "id": "1483253127709706838", "triggers": null }, "sensitive_attributes": [], @@ -217,7 +217,7 @@ { "schema_version": 0, "attributes": { - "id": "7142081680222575747", + "id": "8200555079622067824", "triggers": null }, "sensitive_attributes": [], @@ -290,7 +290,7 @@ "source_file": [], "source_raw": [ { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xVTW/jNhC981cQvOxlqQ9LsRPe0s2iWCxaBJt+AUUPFDmSCdGkOqQce399Qck2oq6DpkVtwJCheU+PM++N5GB+AQzGO0H7sQGpd1l/GzLj833ZQJQV6Y3Tgn5yJn7wrjXdiDIa70jjfQwR5fCT78EFQSjlNKZrQVm/78qtymKF8Oz3OIR6tUbfMkIppRqCQjPE6alMQytHG8+PpxfemWyGxGgFZQUjzmv4Ap1JFRMBodTJHQiqvIvoLR+sdFASShWaJ696iIKOzhxEnuc4ujzVSeMA9YvLLHjVEzqJsBA/HiLKe+ymU1GaHsrNIChbFUUp6k0hNmW5Fish7spZoLJjiIBcuyAoK4usyO7K7KZ6v0S0ohTippoxe2/HHfDBjp1xXBsUlOV+iLk1DRxA5UkNOogQ8pOwU3HIZ2w+lTFivZL2/vHTR6cHb1xMsqXeA0YT4F5rhBBeV98Ypx89RkHXdV0RlWCtUTLCZzgKylqAjargRrVKqg3Um02t1tAqWdTrWtYrfXu3aaui3VRFUdWFXt9sbpsb1Ta1rHRVN4xwzsmbrfZhbuXSbafpPqbhXk5J2e+tLgqRvishyqL4Q6QjMOIgPnvsjetSKwavn8bGJSf8rQdSiHxdv5/mlX7yVZ06EgD3RsEroGmIeVncvj/P+YzTLjz4nTRTBG5DNjoNqKwfdTaNiJ3PYQF/kE52gEkfLO3WWjjw/88ciTKNmMuzEd4J8S7N4wlwf1UBONlY4LLrEDoZPXL0Y0zdpCziCDPpgP5w5MoacJEn1/DWWBA0h7iQN/Qmb9G7yF8iMoXxW5oejieWt9L0cJxoEP4cIcQtSA14USXfqkle9CyJpLX+GTRPSyYI+q2AK5ipm3z+E/iA0JqDoL/xL7DzEfi0XfgVXId+HM44QefPBfZ9unsFNQbApG4BvKB+DoDkagAtxExNIVvmsDzl8PNcs8zhxUNMCEa2IG3cfv3OOH2/uHFah/8UhlPVj0/z24O9uiz5y53KyGzwx8nuDyka/yEZRE3tfkCTMkBZOIYIO83+RZ/kGLfgYlqVp1eRdN4dd35c5EgLmkJDKH2GZut9f+1m4vJovl6odl6DoL/OCPJXAAAA///31/bnqQcAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", + "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", "file_name": "control-plane1-ignition-user-data", "resize": 0 } @@ -610,14 +610,14 @@ [ "2001:470:7116:2::91", "fd00:0:0:2::91", - "fe80::be24:11ff:fed4:dff2" + "fe80::be24:11ff:fe0e:f6cf" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:D4:DF:F2" + "BC:24:11:0E:F6:CF" ], "machine": null, "memory": [ @@ -637,7 +637,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:D4:DF:F2", + "mac_address": "BC:24:11:0E:F6:CF", "model": "virtio", "mtu": 0, "queues": 0, @@ -782,14 +782,14 @@ [ "2001:470:7116:2::92", "fd00:0:0:2::92", - "fe80::be24:11ff:fe27:56ac" + "fe80::be24:11ff:fe4b:b983" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:27:56:AC" + "BC:24:11:4B:B9:83" ], "machine": null, "memory": [ @@ -809,7 +809,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:27:56:AC", + "mac_address": "BC:24:11:4B:B9:83", "model": "virtio", "mtu": 0, "queues": 0, @@ -958,14 +958,14 @@ [ "2001:470:7116:2::93", "fd00:0:0:2::93", - "fe80::be24:11ff:fe1a:a9f2" + "fe80::be24:11ff:fecc:133d" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:1A:A9:F2" + "BC:24:11:CC:13:3D" ], "machine": null, "memory": [ @@ -985,7 +985,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:1A:A9:F2", + "mac_address": "BC:24:11:CC:13:3D", "model": "virtio", "mtu": 0, "queues": 0, @@ -1143,7 +1143,7 @@ "keyboard_layout": "en-us", "kvm_arguments": "", "mac_addresses": [ - "BC:24:11:04:26:47" + "BC:24:11:AC:1F:D9" ], "machine": "", "memory": [ @@ -1163,7 +1163,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:04:26:47", + "mac_address": "BC:24:11:AC:1F:D9", "model": "virtio", "mtu": 0, "queues": 0, @@ -1295,14 +1295,14 @@ [ "2001:470:7116:2::101", "fd00:0:0:2::101", - "fe80::be24:11ff:fe1e:5a3f" + "fe80::be24:11ff:feab:b526" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:1E:5A:3F" + "BC:24:11:AB:B5:26" ], "machine": null, "memory": [ @@ -1322,7 +1322,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:1E:5A:3F", + "mac_address": "BC:24:11:AB:B5:26", "model": "virtio", "mtu": 0, "queues": 0, @@ -1479,14 +1479,14 @@ [ "2001:470:7116:2::102", "fd00:0:0:2::102", - "fe80::be24:11ff:fe1e:f45" + "fe80::be24:11ff:fec8:39fd" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:1E:0F:45" + "BC:24:11:C8:39:FD" ], "machine": null, "memory": [ @@ -1506,7 +1506,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:1E:0F:45", + "mac_address": "BC:24:11:C8:39:FD", "model": "virtio", "mtu": 0, "queues": 0, @@ -1663,14 +1663,14 @@ [ "2001:470:7116:2::103", "fd00:0:0:2::103", - "fe80::be24:11ff:fe20:fede" + "fe80::be24:11ff:febf:b7b5" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:20:FE:DE" + "BC:24:11:BF:B7:B5" ], "machine": null, "memory": [ @@ -1690,7 +1690,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:20:FE:DE", + "mac_address": "BC:24:11:BF:B7:B5", "model": "virtio", "mtu": 0, "queues": 0, @@ -1847,14 +1847,14 @@ [ "2001:470:7116:2::104", "fd00:0:0:2::104", - "fe80::be24:11ff:fe6b:88c7" + "fe80::be24:11ff:fe67:14d8" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:6B:88:C7" + "BC:24:11:67:14:D8" ], "machine": null, "memory": [ @@ -1874,7 +1874,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:6B:88:C7", + "mac_address": "BC:24:11:67:14:D8", "model": "virtio", "mtu": 0, "queues": 0, diff --git a/terraform/terraform.tfstate.backup b/terraform/terraform.tfstate.backup index 3ee6a45..d7810ab 100644 --- a/terraform/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.12.2", - "serial": 1227, + "serial": 1274, "lineage": "d92c42be-29f9-bad9-ef9a-3dc952ff5fa5", "outputs": {}, "resources": [],