From f54296abe9e081ff907eddee94d69e939083b461 Mon Sep 17 00:00:00 2001 From: sebastian Date: Wed, 25 Mar 2026 19:51:19 +0100 Subject: [PATCH] new flatcar version --- terraform/main.tf | 25 + terraform/terraform.tfstate | 1235 +++++++++------- terraform/terraform.tfstate.backup | 2132 +++++++++++++++++++++++++++- 3 files changed, 2859 insertions(+), 533 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 09f2d02..30e0a1a 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -344,6 +344,10 @@ resource "proxmox_virtual_environment_vm" "control_plane1" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.control_plane1_ignition.id}" } + lifecycle { + prevent_destroy = true + ignore_changes = all + } } resource "proxmox_virtual_environment_vm" "control_plane2" { @@ -389,6 +393,10 @@ resource "proxmox_virtual_environment_vm" "control_plane2" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.control_plane2_ignition.id}" } + lifecycle { + prevent_destroy = true + ignore_changes = all + } } resource "proxmox_virtual_environment_vm" "control_plane3" { name = "control-plane3" @@ -433,6 +441,10 @@ resource "proxmox_virtual_environment_vm" "control_plane3" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.control_plane3_ignition.id}" } + lifecycle { + prevent_destroy = true + ignore_changes = all + } } @@ -483,6 +495,10 @@ resource "proxmox_virtual_environment_vm" "worker1" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.worker1_ignition.id}" } + lifecycle { + prevent_destroy = true + ignore_changes = all + } } resource "proxmox_virtual_environment_vm" "worker2" { name = "worker2" @@ -527,6 +543,10 @@ resource "proxmox_virtual_environment_vm" "worker2" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.worker2_ignition.id}" } + lifecycle { + prevent_destroy = true + ignore_changes = all + } } resource "proxmox_virtual_environment_vm" "worker3" { name = "worker3" @@ -615,4 +635,9 @@ resource "proxmox_virtual_environment_vm" "worker4" { datastore_id = "Pool1" user_data_file_id = "${proxmox_virtual_environment_file.worker4_ignition.id}" } + + lifecycle { + prevent_destroy = true + ignore_changes = all + } } diff --git a/terraform/terraform.tfstate b/terraform/terraform.tfstate index d6ae7d5..710571b 100644 --- a/terraform/terraform.tfstate +++ b/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.12.2", - "serial": 1593, + "terraform_version": "1.14.7", + "serial": 1610, "lineage": "d92c42be-29f9-bad9-ef9a-3dc952ff5fa5", "outputs": {}, "resources": [ @@ -14,10 +14,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 755\n - path: /etc/kubernetes/manifests\n #overwrite: true\n mode: 0755\n - path: /etc/install-calico\n overwrite: true\n mode: 0755\n - path: /var/lib/undercloud-stamps\n mode: 0755\n\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::91/64\n Address=2001:470:7116:2::91/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.91/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /opt/bin/velero\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\"\n\n - path: /etc/kubernetes/kubeadm-init.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n bootstrapTokens:\n - token: \"kvg1hc.t3rewovrps426rof\"\n description: \"default kubeadm bootstrap token\"\n ttl: \"0\"\n nodeRegistration:\n name: control-plane1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::91\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::91\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n controlPlaneEndpoint: \"[fd00:0:0:2::100]:6443\"\n networking:\n podSubnet: \"2001:470:7116:a::/64,10.0.10.0/24\"\n serviceSubnet: \"2001:470:7116:f:1::/108,10.0.91.0/24\"\n dnsDomain: \"k8s.undercloud.local\"\n controllerManager:\n extraArgs:\n flex-volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n bind-address: '::'\n apiServer:\n extraArgs:\n enable-aggregator-routing: \"true\"\n proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt\n proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key\n requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt\n requestheader-allowed-names: front-proxy-client\n requestheader-extra-headers-prefix: X-Remote-Extra-\n requestheader-group-headers: X-Remote-Group\n requestheader-username-headers: X-Remote-User\n\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec\n cgroupDriver: \"systemd\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\n - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: v1\n kind: Service\n metadata:\n name: kube-dns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n spec:\n type: ClusterIP\n ipFamilyPolicy: RequireDualStack\n ipFamilies: [IPv6, IPv4]\n clusterIP: 2001:470:7116:f:1::53\n clusterIPs:\n - 2001:470:7116:f:1::53\n - 10.0.91.53\n ports:\n - name: dns\n port: 53\n protocol: UDP\n targetPort: 53\n - name: dns-tcp\n port: 53\n protocol: TCP\n targetPort: 53\n - name: metrics\n port: 9153\n protocol: TCP\n targetPort: 9153\n selector:\n k8s-app: kube-dns\n\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"setting timezone to Europe/Berlin\"'\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-init.service\n enabled: true\n contents: |\n [Unit]\n Description=Kubeadm Init Cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n ConditionPathExists=!/etc/kubernetes/kubelet.conf\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/opt/bin/kubeadm reset -f\n\n ExecStart=/bin/sh -c 'echo \"kubeadm-init.service started...\"'\n\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"running kubeadm init...\"'\n ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\n \n # copy files for kubectl\n ExecStart=/bin/sh -c 'echo \"copying files (admin.conf) to core home folder.\"'\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n - name: install-calico.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\n\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.calico.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create calico namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\n ExecStart=/bin/sh -c 'echo \"install tigera operator...\"'\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 60s\n ExecStart=/bin/sh -c 'echo \"witing for tigera operator... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"create clico custom ressources...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\n \n ExecStart=/bin/sh -c 'echo \"witing 3m..\"'\n ExecStart=/bin/sleep 3m\n #ExecStart=/bin/sh -c 'echo \"apply calico (calico-apiserver)...\"'\n #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\n #ExecStart=/bin/sh -c 'echo \"witing 1m...\"'\n #ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"witing calico-apiserver... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"witing 120s...\"'\n ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"apply calico-peers...\"'\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 1m\n ExecStart=/bin/sh -c 'echo \"apply calico-ippools...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\n \n #ExecStart=/bin/sh -c 'echo \"witing for whisker..\"'\n #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\n #ExecStart=/bin/sh -c 'echo \"port-forward -n calico-system service/whisker 8081:8081\"'\n #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\n ExecStart=/usr/bin/systemctl disable install-calico.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-ceph.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.ceph.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create ceph namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\n\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\n \n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\n ExecStart=/usr/bin/systemctl disable install-ceph.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-gitea.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-ceph.service\n After=install-ceph.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\n \n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sleep 4m\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\n ExecStart=/bin/sleep 60s\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\n ExecStart=/bin/sleep 3m\n ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\n ExecStart=/usr/bin/systemctl disable install-gitea.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-argocd.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n\n ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\n \n ExecStart=/bin/sleep 1m\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\n ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\n\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\n\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\n\n\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\n ##ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\n #ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\n ExecStart=/usr/bin/systemctl disable install-argocd.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: pin-service-ips.service\n enabled: true\n contents: |\n [Unit]\n Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\n After=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\n\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n ExecStart=/bin/sh -eu -c '\\\n echo \"[pin-service-ips] waiting for API...\" ; \\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \u003e/dev/null 2\u003e\u00261 \u0026\u0026 break; sleep 2; done ; \\\n echo \"[pin-service-ips] ensure namespaces exist...\" ; \\\n kubectl get ns kube-system \u003e/dev/null ; \\\n kubectl get ns argocd \u003e/dev/null 2\u003e\u00261 || kubectl create ns argocd ; \\\n kubectl get ns calico-system \u003e/dev/null ; \\\n echo \"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\" ; \\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\n echo \"[pin-service-ips] replace Services with fixed ClusterIPs...\" ; \\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\n echo \"[pin-service-ips] done.\" \\\n '\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n", - "id": "374759146", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 755\n - path: /etc/kubernetes/manifests\n #overwrite: true\n mode: 0755\n - path: /etc/install-calico\n overwrite: true\n mode: 0755\n - path: /var/lib/undercloud-stamps\n mode: 0755\n\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::91/64\n Address=2001:470:7116:2::91/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.91/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /opt/bin/velero\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\"\n\n - path: /etc/kubernetes/kubeadm-init.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n bootstrapTokens:\n - token: \"kvg1hc.t3rewovrps426rof\"\n description: \"default kubeadm bootstrap token\"\n ttl: \"0\"\n nodeRegistration:\n name: control-plane1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::91\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::91\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n controlPlaneEndpoint: \"[fd00:0:0:2::100]:6443\"\n networking:\n podSubnet: \"2001:470:7116:a::/64,10.0.10.0/24\"\n serviceSubnet: \"2001:470:7116:f:1::/108,10.0.91.0/24\"\n dnsDomain: \"k8s.undercloud.local\"\n controllerManager:\n extraArgs:\n flex-volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n bind-address: '::'\n apiServer:\n extraArgs:\n enable-aggregator-routing: \"true\"\n proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt\n proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key\n requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt\n requestheader-allowed-names: front-proxy-client\n requestheader-extra-headers-prefix: X-Remote-Extra-\n requestheader-group-headers: X-Remote-Group\n requestheader-username-headers: X-Remote-User\n\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec\n cgroupDriver: \"systemd\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\n - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: v1\n kind: Service\n metadata:\n name: kube-dns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n spec:\n type: ClusterIP\n ipFamilyPolicy: RequireDualStack\n ipFamilies: [IPv6, IPv4]\n clusterIP: 2001:470:7116:f:1::53\n clusterIPs:\n - 2001:470:7116:f:1::53\n - 10.0.91.53\n ports:\n - name: dns\n port: 53\n protocol: UDP\n targetPort: 53\n - name: dns-tcp\n port: 53\n protocol: TCP\n targetPort: 53\n - name: metrics\n port: 9153\n protocol: TCP\n targetPort: 9153\n selector:\n k8s-app: kube-dns\n\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"setting timezone to Europe/Berlin\"'\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-init.service\n enabled: true\n contents: |\n [Unit]\n Description=Kubeadm Init Cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n ConditionPathExists=!/etc/kubernetes/kubelet.conf\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/opt/bin/kubeadm reset -f\n\n ExecStart=/bin/sh -c 'echo \"kubeadm-init.service started...\"'\n\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"running kubeadm init...\"'\n ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\n \n # copy files for kubectl\n ExecStart=/bin/sh -c 'echo \"copying files (admin.conf) to core home folder.\"'\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n - name: install-calico.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\n\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.calico.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create calico namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\n ExecStart=/bin/sh -c 'echo \"install tigera operator...\"'\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 60s\n ExecStart=/bin/sh -c 'echo \"witing for tigera operator... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"create clico custom ressources...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\n \n ExecStart=/bin/sh -c 'echo \"witing 3m..\"'\n ExecStart=/bin/sleep 3m\n #ExecStart=/bin/sh -c 'echo \"apply calico (calico-apiserver)...\"'\n #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\n #ExecStart=/bin/sh -c 'echo \"witing 1m...\"'\n #ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"witing calico-apiserver... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"witing 120s...\"'\n ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"apply calico-peers...\"'\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 1m\n ExecStart=/bin/sh -c 'echo \"apply calico-ippools...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\n \n #ExecStart=/bin/sh -c 'echo \"witing for whisker..\"'\n #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\n #ExecStart=/bin/sh -c 'echo \"port-forward -n calico-system service/whisker 8081:8081\"'\n #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\n ExecStart=/usr/bin/systemctl disable install-calico.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-ceph.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\n\n [Service]\n Type=oneshot\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.ceph.service started...\"'\n ExecStart=/bin/sleep 30\n\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\n\n # this must NOT be ignored\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\n\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\n\n # readiness gates (only stamp if these succeed)\n ExecStart=/opt/bin/kubectl -n ceph rollout status deploy/csi-cephfsplugin-provisioner --timeout=5m\n ExecStart=/opt/bin/kubectl -n ceph rollout status ds/csi-cephfsplugin --timeout=5m\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\n ExecStartPost=-/usr/bin/systemctl disable install-ceph.service\n\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n - name: install-gitea.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-ceph.service\n After=install-ceph.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\n \n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sleep 4m\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\n ExecStart=/bin/sleep 60s\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\n ExecStart=/bin/sleep 3m\n ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\n ExecStart=/usr/bin/systemctl disable install-gitea.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-argocd.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n\n ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\n \n ExecStart=/bin/sleep 1m\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\n ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\n\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\n\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\n\n\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\n ##ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\n #ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\n ExecStart=/usr/bin/systemctl disable install-argocd.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: pin-service-ips.service\n enabled: true\n contents: |\n [Unit]\n Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\n After=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\n\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n ExecStart=/bin/sh -eu -c '\\\n echo \"[pin-service-ips] waiting for API...\" ; \\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \u003e/dev/null 2\u003e\u00261 \u0026\u0026 break; sleep 2; done ; \\\n echo \"[pin-service-ips] ensure namespaces exist...\" ; \\\n kubectl get ns kube-system \u003e/dev/null ; \\\n kubectl get ns argocd \u003e/dev/null 2\u003e\u00261 || kubectl create ns argocd ; \\\n kubectl get ns calico-system \u003e/dev/null ; \\\n echo \"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\" ; \\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\n echo \"[pin-service-ips] replace Services with fixed ClusterIPs...\" ; \\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\n echo \"[pin-service-ips] done.\" \\\n '\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "2362473131", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/opt/bin/kubeadm reset -f\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQWuEMBAF4Pv8jd56yGay4rKBHIQtpYeKtEfxEJIpSjUpSdT674u21ULJ6b2Pl6mfdTJtA6UeSFFq7wHqktLsw3sDhbWBYlRvlnO5PiHlFU95tovgHGV24fKCmB9896gTzXr572f4pb+fnuFWvh4NSon7CeSMM8GueBLZPv4pv3dbQIZw84PuXFSjsxRM70fLem90D0/VlBfG0Ed6KZTzW65CN2mzPHwmcrHzLq7wFQAA//9FuT+UDwEAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/opt/bin/kubeadm reset -f\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nType=oneshot\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nExecStart=/bin/sleep 30\\n\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\n# this must NOT be ignored\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\n\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\n# readiness gates (only stamp if these succeed)\\nExecStart=/opt/bin/kubectl -n ceph rollout status deploy/csi-cephfsplugin-provisioner --timeout=5m\\nExecStart=/opt/bin/kubectl -n ceph rollout status ds/csi-cephfsplugin --timeout=5m\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStartPost=-/usr/bin/systemctl disable install-ceph.service\\n\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -35,10 +35,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::92/64\n Address=2001:470:7116:2::92/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.92/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8s.undercloud.local/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"fd00:0:0:2::92\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::92\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubeadm.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "3116130501", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::92/64\n Address=2001:470:7116:2::92/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.92/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8s.undercloud.local/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"fd00:0:0:2::92\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::92\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubeadm.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target", + "id": "1638069321", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBU+y9DScMRBpzkQOkF36XUVc1fasX0lVf189EALOFjDKKKcFT/hVeDzvuwOAAqncpAYb5GwNej262dDRaTWSj2bJSq3xL36V0rqDGz8sSm9v/xFtGJwNu7gFAAD//0NI5lj4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwWobQQyG7/sUOjoBj68lN5MGWmgTaJJDe5M1ylp4drRImoXt0xd7c+jBNYX4v8xISB/fMLcfT3cL/8j29eXL04+vvx4+w/b+/uH5GZ4ev/08M3iB8d85MV724uCzBw8gDiNagL4BgjM14wxS3ww9rFE043SOsS0FkEImCWEHNIZBq4Qe97FmKNr3nNMlj9eKLfZq8vu4Q8TuoAaDeHM+mnmYUJQZRtO97CTe2X8xBpzB2FsJkApZnGQsUtHmxYJ7XES1pvMeV/jbbn3ldIe2Y6sc7EBaw7QwjAUrw6Nm7rrvWLFnmATvTrMUBVbvl5uOsAgprJbz1Jq4sCmsYYd0aCOslsZNh9YrZdhHjH632Szl2tkmtrRUySdKh0+eWs1sVLTlVJSwbK7/8D8BAAD//2cEMuhrAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSv27bPBTFdz4FofmTRFmy5XDzl2RoAxRGXaRD0YEir2xCMq/AP0qcpy8oqakddCu8GOI55L3nd8Sgn8E6jYbTLjQg1Dnrti7TmI9FA16UpNNGcfoZtblH0+pjsMJrNESi8Rb7fS8McEJpj1L0u/2nR6MG1MbHb5QKNYL12sFOKQvOcZq0ijEefyvO71bJJGu0UXu0ntNNVZWEUhldrZbCwxNcogugliWsZSuFrKGq60puoJWCVZtKVCu1vavbkrV1yVhZMbVZ19tmLdumEqUqqyYhBhV8haN2ft4gDmjEGThdVkmHuMsqvm71AWUHntNg9CvP89wGk0ed0AasuvqbOZQdoVN8PfjHV2/Fzh7dvH98NNUDp8mKsYJXNeN1UWyud5d9cB5sqkxMp2AZy+6KbF3+d+toecH5upw9I/bhDOnQh6M2qdKW0yTHwee9buAVZB6nsQY8uHwZbBG7fPbmkywhSjuJI9hLnLdB9DGe4Rt2YBaCgz6AHcG+g6XJj2uGBWM/ecQ2j+YnK0268VicZOZLCy842sFVq43FdhYF40QLh04P97tnsDPpqYXeBiBpmpKP1ezBZ3Jq4G1Di6WhT7PmtqTivXWcJ+QEovent/+1UbubgwXBA56FnmbfuiwYBVb2GFQ2VfuP6sshJpN+RHoFKL3mmBB5tBiGB6tHiKDcxXk4q4SSGcV+AvPwDxBF8Ccw/neIhFJh0FzOGJYaghFND2qJl9IXaE6I3d8O411o9dv7VWdUwOn32UF+BQAA//8OJfhXMgQAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubeadm.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQWuEMBAF4Pv8jd56yGay4rKBHIQtpYeKtEfxEJIpSjUpSdT674u21ULJ6b2Pl6mfdTJtA6UeSFFq7wHqktLsw3sDhbWBYlRvlnO5PiHlVZzybBfBOcrswuUFMT/47lEnmvXy38/wS38/PcOtfD0alBL3E8gZZ4JdxUlk+/in/N5tARnCzQ+6c1GNzlIwvR8t673RPTxVU14YQx/ppVDOb7kK3aTN8vCZyMXOu7jCVwAAAP//baxNNg8BAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwWobQQyG7/sUOjoBj68lN5MGWmgTaJJDe5M1ylp4drRImoXt0xd7c+jBNYX4v8xISB/fMLcfT3cL/8j29eXL04+vvx4+w/b+/uH5GZ4ev/08M3iB8d85MV724uCzBw8gDiNagL4BgjM14wxS3ww9rFE043SOsS0FkEImCWEHNIZBq4Qe97FmKNr3nNMlj9eKLfZq8vu4Q8TuoAaDeHM+mnmYUJQZRtO97CTe2X8xBpzB2FsJkApZnGQsUtHmxYJ7XES1pvMeV/jbbn3ldIe2Y6sc7EBaw7QwjAUrw6Nm7rrvWLFnmATvTrMUBVbvl5uOsAgprJbz1Jq4sCmsYYd0aCOslsZNh9YrZdhHjH632Szl2tkmtrRUySdKh0+eWs1sVLTlVJSwbK7/8D8BAAD//2cEMuhrAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSv27bPBTFdz4FofmTRFmy5XDzl2RoAxRGXaRD0YEir2xCMq/AP0qcpy8oqakddCu8GOI55L3nd8Sgn8E6jYbTLjQg1Dnrti7TmI9FA16UpNNGcfoZtblH0+pjsMJrNESi8Rb7fS8McEJpj1L0u/2nR6MG1MbHb5QKNYL12sFOKQvOcZq0ijEefyvO71bJJGu0UXu0ntNNVZWEUhldrZbCwxNcogugliWsZSuFrKGq60puoJWCVZtKVCu1vavbkrV1yVhZMbVZ19tmLdumEqUqqyYhBhV8haN2ft4gDmjEGThdVkmHuMsqvm71AWUHntNg9CvP89wGk0ed0AasuvqbOZQdoVN8PfjHV2/Fzh7dvH98NNUDp8mKsYJXNeN1UWyud5d9cB5sqkxMp2AZy+6KbF3+d+toecH5upw9I/bhDOnQh6M2qdKW0yTHwee9buAVZB6nsQY8uHwZbBG7fPbmkywhSjuJI9hLnLdB9DGe4Rt2YBaCgz6AHcG+g6XJj2uGBWM/ecQ2j+YnK0268VicZOZLCy842sFVq43FdhYF40QLh04P97tnsDPpqYXeBiBpmpKP1ezBZ3Jq4G1Di6WhT7PmtqTivXWcJ+QEovent/+1UbubgwXBA56FnmbfuiwYBVb2GFQ2VfuP6sshJpN+RHoFKL3mmBB5tBiGB6tHiKDcxXk4q4SSGcV+AvPwDxBF8Ccw/neIhFJh0FzOGJYaghFND2qJl9IXaE6I3d8O411o9dv7VWdUwOn32UF+BQAA//8OJfhXMgQAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubeadm.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -56,10 +56,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::93/64\n Address=2001:470:7116:2::93/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.93/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::93\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::93\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "3301975261", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::93/64\n Address=2001:470:7116:2::93/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.93/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::93\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::93\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target", + "id": "1600146304", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuHMBBH93yUDjEXRTGQQbCUDhVpR3EIyRWlmpQkav32RVt0+HPTu8fvdW8q6qEnjZpRYhyeCOkajJvzXz2pjPEYgvw0jInjuBBlmuTZZThjILKCiQIgv/WLirip/VGnpG4+7h4IAVcLGGWU0zJN+F34f/7tTgAKpHazGm2QizXo9eQWQyen1URe2zWvtMbv+F5J605u/bgqvT//RLRhdDYc4jcAAP//AgBuPPgAAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSP2/bMBDFd34KQnMlUZZsOdzcJEMboDDqIh2KDhR5sgnJPIF/lDifvqCkpnFQdCm8GOI78t37PTHoR7BOo+G0Cw0Idc66rcs05mPRgBcl6bRRnH5GbW7RtPoYrPAaDZFovMV+3wsDnFDaoxT9bv/p3qgBtfHxG6VCjWC9drBTyoJznCYrxgpe1YzXRbHhK85vymTSNtqoPVrP6aaqSkKpjKOtlsLDA1w4TVqAWpawlq0Usoaqriu5gVYKVm0qUa3U9qZuS9bWJWNlxdRmXW+btWybSpSqrJqEGFTwFY7a+XmN6NKIM3C67JMOcaHpdasPKDvwnAajn3me5zaYPOqENmDVm7+ZQ9kROmXYg79/9lbs7NHNIcRHUz38a3fZB+fBpsrEiAqWseymyNblh+uJlhecr5eZEftwhnTow1GbVGnLaZLj4PNeN/AMMo9urAEPLl+MLWKXz7P5JEuI0k7iCPYS/TaIPsYzfMMOzIJx0AewI9hXujT50SrGePytOC8Y+8kjttman0Zp0o3H4iQzX1p4wtEOrlptLLazKBgnWjh0erjdPYKdSU9V9DYASdOUvO9nDz6TUw2va1osNX2YNddNFa/V4zwhJxC9P7181Ebtrg4WBHd4FnryvnVZMAqs7DGobOr3H9WXQ0wmfY/0DaD0LceEyKPFMNxZPUIE5S7Ow1kllMwo9hOYu/+AKII/gfG/QySUCoPmcsaw1BCMaHpQS7yUPkFzQuz+dhjvQqtfXq86owJOv88T5FcAAAD//xliG4M3BAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQUvEMBAF4Pv8DW8espmkdNlADoUV8WBZ9LjsISQjLbaJJGlr/720aitITu99vMz12WTb3KA2PWnKzT3AtaY8hfh+g8q5SCnpN8e5Wp5Q6iQPZbGJ4BxVceTqiFjufPdoMk1m/u8SfunvpxLO9eveoFK4nUDOOBPsJA+i2MY/5fduDcgQzqE3rU968I6i7cLgWBes6eDpMpaVtfSRXyrtw5ovsR2NnR8+M/nUBp8W+AoAAP//dV+cVw8BAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSP2/bMBDFd34KQnMlUZZsOdzcJEMboDDqIh2KDhR5sgnJPIF/lDifvqCkpnFQdCm8GOI78t37PTHoR7BOo+G0Cw0Idc66rcs05mPRgBcl6bRRnH5GbW7RtPoYrPAaDZFovMV+3wsDnFDaoxT9bv/p3qgBtfHxG6VCjWC9drBTyoJznCYrxgpe1YzXRbHhK85vymTSNtqoPVrP6aaqSkKpjKOtlsLDA1w4TVqAWpawlq0Usoaqriu5gVYKVm0qUa3U9qZuS9bWJWNlxdRmXW+btWybSpSqrJqEGFTwFY7a+XmN6NKIM3C67JMOcaHpdasPKDvwnAajn3me5zaYPOqENmDVm7+ZQ9kROmXYg79/9lbs7NHNIcRHUz38a3fZB+fBpsrEiAqWseymyNblh+uJlhecr5eZEftwhnTow1GbVGnLaZLj4PNeN/AMMo9urAEPLl+MLWKXz7P5JEuI0k7iCPYS/TaIPsYzfMMOzIJx0AewI9hXujT50SrGePytOC8Y+8kjttman0Zp0o3H4iQzX1p4wtEOrlptLLazKBgnWjh0erjdPYKdSU9V9DYASdOUvO9nDz6TUw2va1osNX2YNddNFa/V4zwhJxC9P7181Ebtrg4WBHd4FnryvnVZMAqs7DGobOr3H9WXQ0wmfY/0DaD0LceEyKPFMNxZPUIE5S7Ow1kllMwo9hOYu/+AKII/gfG/QySUCoPmcsaw1BCMaHpQS7yUPkFzQuz+dhjvQqtfXq86owJOv88T5FcAAAD//xliG4M3BAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -77,10 +77,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::101/64\n Address=2001:470:7116:2::101/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.101/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker1.undercloud.local worker1\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker1.undercloud.local worker1\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker1.undercloud.local worker1\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::101\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "1551140390", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::101/64\n Address=2001:470:7116:2::101/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.101/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::101\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "393379439", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASAR/z7FQCAGVWgCwQ8xv/ohOterv3T6yqP64iSolnDIEDF3xPiCvx//0bHoAcWeUnPbioZmcpmNHPlo/e6JG9NkteGkPf6b1Uzh/chGHRZnv+SeTi4F3cxW8AAAD//9kKqIr7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5yTbW6EIBBA/3MKLrBkPpo1ndtYtGmzRAzdTa/fYNEopaU4/sLhvcSXiNQZMGBQa+287d2b/7grkXj+nu2tfp+vl/zk55fe3pQiAJSnDqRDvAqJPKPW1k/34N1ldv00onlMwxis84/BLJpsX3BQ5qCKgwoOzhxccfBPBwLqTx9uYyh8RFqUKDpFcVrybxQr9ToASHxS6TiNtQ8KKijqsQ8KLijqrfeKmDlONdoRojMQ76C/QmP8OcgsjRsDryi1h11Rbg+a0KVltcl2mVou8z/CfQUAAP//Msgv7lgEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRz27bPBDE73oKQudPf2g7Tr69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+dnBnpFCcQeVJeOaNy57J5CSVyN+ojRrLOOvAP1mck/s2+oTWIisc88O/yOLYW4HECmlDdnBPXG0qHoTCkrdGDbYQSVPF2gqipJvrLsoyGP4m62ZWDbZWqO0WN8uUQxO2nD9K5S028FDaDyVV1r2DzW8Kj1FlYAutb5rLF9ChGlcD6AynVd1uX/unxY/3dvaUADPKwXz8h9OmMx9KklXzgSUHnFQ6x6OuIFbTXFEY8RQ3VNdhWHavFWsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV9cwrXmQ+hdsN5trtDhbVd6NrT7ZMq4F33iUIWxWW+FmESUfTIOHjobn3SsKNWQXGipKwqwoiuwj4x5jaWeU96j1FfWXRXNP2zgnGKZeAfLshKaPp9+fyLvd3cUVwZ7PhubsT6FM3qHYnpMre7am/6v6epiaKT4yvQFU3HLMM9sKp2EvNOIEKryHiGeXq2xB8W0Gs/8HiH8CAAD//1pu0RwQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nShUAOhRXxYFn0uOwhJCMttokkaWv/vbRqLSxzeu+b997lxWTbXKE2PWvOzT3ApeY8hfhxhcq5yCnpd4eolpNKEdJDWWxIIpIqjqiOROWO3z2ZzJOZbx8O8If2tQc41W//DilF2wahQCHF0iyLLf3r/gRXQYLgFHrT+qQH7zjaLgxOdMGaDp7PY1lZy5/5tdI+rPoc29HY+fErs09t8GkB3wEAAP//4pHcSRIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRz27bPBDE73oKQudPf2g7Tr69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+dnBnpFCcQeVJeOaNy57J5CSVyN+ojRrLOOvAP1mck/s2+oTWIisc88O/yOLYW4HECmlDdnBPXG0qHoTCkrdGDbYQSVPF2gqipJvrLsoyGP4m62ZWDbZWqO0WN8uUQxO2nD9K5S028FDaDyVV1r2DzW8Kj1FlYAutb5rLF9ChGlcD6AynVd1uX/unxY/3dvaUADPKwXz8h9OmMx9KklXzgSUHnFQ6x6OuIFbTXFEY8RQ3VNdhWHavFWsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV9cwrXmQ+hdsN5trtDhbVd6NrT7ZMq4F33iUIWxWW+FmESUfTIOHjobn3SsKNWQXGipKwqwoiuwj4x5jaWeU96j1FfWXRXNP2zgnGKZeAfLshKaPp9+fyLvd3cUVwZ7PhubsT6FM3qHYnpMre7am/6v6epiaKT4yvQFU3HLMM9sKp2EvNOIEKryHiGeXq2xB8W0Gs/8HiH8CAAD//1pu0RwQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -98,10 +98,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::102/64\n Address=2001:470:7116:2::102/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.102/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::102\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "2705472928", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::102/64\n Address=2001:470:7116:2::102/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.102/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::102\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "3758288308", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASQTzm2akEAMqsAFkg5jf+RSda9Xbvn1hVf1xFlBLPGAIHLvieEFfi//s3PAA5sspPenBRzc5SMKOfLR+90SN7bZa8NIa+03upnD+4CcOizfb8k8jFwbu4i98AAAD//8ZKOy/7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRwW7bPBCE73oKQudfomQ7Tv69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+czA72iBGIPqktHNO5cdk+hJNZjfcRo1llH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VBWmVJW6MC2wwgqebqA1lqS15Z9NORR3M22DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVS+qqoaNo8VPNb1FlYAdbXKZ43tU4gohfMBVF5XZVX+X5cP6//uLQ3UAA/rxTNyn85YDH1qyReOBFSueYi6pyNe0OopjniMGPQ12VUc9OLVsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV1UwrXmQ6hdsN5trtDhbVd6NbX2yZVwLvvEoQ9istsLNIko+mAYPHQ3Pu1cUasguNFSUhFlRFNlHxj3G0s4o71HXV9RfFs09beOcYJh6BcizE5o+nn5/Iu92dxdXBHs+G5qzP4UyeYdie06u7Nma/q/q62FqpvjI9AZQccsxz2wrnIa90IgTqPAeIp5drrIFxbcZzP4fIP4JAAD//3XfDwQQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nShUAOhRXxYFn0uOwhJCMttokkaWv/vbRqLSxzeu+b997lxWTbXKE2PWvOzT3ApeY8hfhxhcq5yCnpd4eolpNKEcqHstiQRCRVHFEdicodv3symScz3z4c4A/taw9wqt/+HVKKtg1CgUKKpVkWW/rX/QmuggTBKfSm9UkP3nG0XRic6II1HTyfx7Kylj/za6V9WPU5tqOx8+NXZp/a4NMCvgMAAP//yrsJeBIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRwW7bPBCE73oKQudfomQ7Tv69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+czA72iBGIPqktHNO5cdk+hJNZjfcRo1llH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VBWmVJW6MC2wwgqebqA1lqS15Z9NORR3M22DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVS+qqoaNo8VPNb1FlYAdbXKZ43tU4gohfMBVF5XZVX+X5cP6//uLQ3UAA/rxTNyn85YDH1qyReOBFSueYi6pyNe0OopjniMGPQ12VUc9OLVsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV1UwrXmQ6hdsN5trtDhbVd6NbX2yZVwLvvEoQ9istsLNIko+mAYPHQ3Pu1cUasguNFSUhFlRFNlHxj3G0s4o71HXV9RfFs09beOcYJh6BcizE5o+nn5/Iu92dxdXBHs+G5qzP4UyeYdie06u7Nma/q/q62FqpvjI9AZQccsxz2wrnIa90IgTqPAeIp5drrIFxbcZzP4fIP4JAAD//3XfDwQQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -119,10 +119,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::103/64\n Address=2001:470:7116:2::103/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.103/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::103\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "4294157633", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::103/64\n Address=2001:470:7116:2::103/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.103/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::103\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "4172286456", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLd/KIQyEGwlB4q0h7FQ0i2KNWkJFHr2xdt0cLPnmaHb9pXnUzfsVpPpCj1D4y1NaXVh8+OldYGilF9WAC5n5AS4faYZ6cSACizAmSBmP/zzzrRqrd7f2NV/X4VUUo8YwgcuOB7QlyJv+/v8ADkyCo/6cFFNTtLwYx+tnz0Ro/spVny0hj6Sm+lcv7gJgyLNtvTdyIXB+/iLn4CAAD//zN1Skz7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRPY/bPBCEe/0KQvUrUTr7Pt7tnPM1CRAEcXBNkIImV/JCMldYkjpffn2gDyD2tQEbgpwhZ+cxA72iBGIPqktHNO5cdk+hJNZjfcRoNllH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VA2mVJW6MC2wwgqebqA1lqS15Z9NORR3NW2DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVR+V1U1bB8reKzrB7gDqKtNPmtsn0JEKZwPoPK6Kqvy/7q83/x3a2mgBrhfPSP36YzF0KeWfOFIQOWah6h7OuIFrZ7iiMeIQa/JVnHQi1fPsjxzFCyPKO9T4CNznIoZfnCHfhnBDHRAGVFevBuYfASV/2xcVcG05kGqX/Cw3a7R4mxVeTe29cmWcSP4xqMMYXv3INwsouSDafDQ0fC8e0WhhuxCQ0VJmBVFkX1k3GMs7YzyFnW9ov6yaG5pG+cEw9QrQJ6d0PTx9PsTebe7uVgR7PlsaM7+FMrkHYrtObmyZ2v6v6qvh6mZ4iPTK0DFNcc8s61wGvZCI06gwnuIeHa5yhYU32Yw+3+A+CcAAP//r02VuhADAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM2npQiCHwop4sCx6XPYQkpEttokkaWv/vbRqLSxzeu+b9975xWR7vUBjetacr/cA54bzFOLHBWrnIqek3x2iWk4qRVg8VOWGJCKp8oDqQFTt+N2TyTyZ+fahgD+0ry3g2Lz9O6QUbRuEAoUUS7Mst/Sv+xNcBQmCY+hN65MevONouzA40QVrOng+jVVtLX/m11r7sOpTbEdj58evzD61wacFfAcAAP//7V9q3hIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRPY/bPBCEe/0KQvUrUTr7Pt7tnPM1CRAEcXBNkIImV/JCMldYkjpffn2gDyD2tQEbgpwhZ+cxA72iBGIPqktHNO5cdk+hJNZjfcRoNllH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VA2mVJW6MC2wwgqebqA1lqS15Z9NORR3NW2DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVR+V1U1bB8reKzrB7gDqKtNPmtsn0JEKZwPoPK6Kqvy/7q83/x3a2mgBrhfPSP36YzF0KeWfOFIQOWah6h7OuIFrZ7iiMeIQa/JVnHQi1fPsjxzFCyPKO9T4CNznIoZfnCHfhnBDHRAGVFevBuYfASV/2xcVcG05kGqX/Cw3a7R4mxVeTe29cmWcSP4xqMMYXv3INwsouSDafDQ0fC8e0WhhuxCQ0VJmBVFkX1k3GMs7YzyFnW9ov6yaG5pG+cEw9QrQJ6d0PTx9PsTebe7uVgR7PlsaM7+FMrkHYrtObmyZ2v6v6qvh6mZ4iPTK0DFNcc8s61wGvZCI06gwnuIeHa5yhYU32Yw+3+A+CcAAP//r02VuhADAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -140,10 +140,10 @@ { "schema_version": 0, "attributes": { - "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker4\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::104/64\n Address=2001:470:7116:2::104/64\n Gateway=2001:470:7116:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.104/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker4\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::104\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n", - "id": "1066423053", + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker4\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::104/64\n Address=2001:470:7116:2::104/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.104/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker4\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::104\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "1542817396", "pretty_print": true, - "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker4%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASIXvMs1MJAJRZAbJAzG/8i0606u3eP7Gq/riKKCWeMQQOXPA9Ia7E//dveAByZJWf9OCimp2lYEY/Wz56o0f22ix5aQx9p/dSOX9wE4ZFm+35J5GLg3dxF78BAAD//7nMbL/7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRv47bPBDEez0FofqTKJ11f77tnPM1CRAEcXBNkIImV/JCMldYkjpfnj6QZCD2tQEbgpwhZ+dnRnpFCcQeVJ8OaNyp7J9CSayn+oDRbLKevAP1mck/s2+pS2Iisc88O/yOHYW4HkCmlDcnBPXG0qM0mVJWaM+2xwgqeTqD1lqS15Z9NORR3NW2DGz7TC0xBowv5yhmK12Y31Vq/q2gEVR+V1U1NI8VPNb1A9wB1FWTLxo7pBBRCucDqLyuyqr8vy7vN//dWlqoAe43q2fiIZ2wGIfUkS8cCahc8xj1QAc8o9VzHPEYMehLsos46NWrF1meOQqWJ5T3OfCBOc7FjD+4R7+OYEbao0woL96NTD6Cyn+2rqpgXssg1S94aJpLtLhYVd5PXX20ZdwIvvEkY2juHoTbVZR8MC3uexqft68o1JJdaagoCbOiKLKPjAeMpV1Q3qKuL6i/rJpb2sY5wTD3CpBnRzRDPP7+RN5tby4uCHZ8MrRkfwpl8g7FDpxcObA1w1/V1/3cTPGR6RWg4ppjntlOOI07oQlnUOE9RDy5XGUrim8LmN0/QPwTAAD//yu9sjUQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker4%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nQhUAOhRXxYFn0uOwhJCNb7CaSpK3999KqtbDM6b1v3nunF1vc5QyNvbLhcrkHODVcxpg+zlB7nzhn8+4R9XxSa0L1UKkVSUTSao96T1Rt+N2TLTza6fZhB39oW7uDQ/P275DWtG4QChRSzM1Srelf9ye4CBIEh3i1bcimD56T62LvRRed7eD5OFS1c/xZXmsT4qKPqR2smx6/CofcxpBn8B0AAP//mu+jGxIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRv47bPBDEez0FofqTKJ11f77tnPM1CRAEcXBNkIImV/JCMldYkjpfnj6QZCD2tQEbgpwhZ+dnRnpFCcQeVJ8OaNyp7J9CSayn+oDRbLKevAP1mck/s2+pS2Iisc88O/yOHYW4HkCmlDcnBPXG0qM0mVJWaM+2xwgqeTqD1lqS15Z9NORR3NW2DGz7TC0xBowv5yhmK12Y31Vq/q2gEVR+V1U1NI8VPNb1A9wB1FWTLxo7pBBRCucDqLyuyqr8vy7vN//dWlqoAe43q2fiIZ2wGIfUkS8cCahc8xj1QAc8o9VzHPEYMehLsos46NWrF1meOQqWJ5T3OfCBOc7FjD+4R7+OYEbao0woL96NTD6Cyn+2rqpgXssg1S94aJpLtLhYVd5PXX20ZdwIvvEkY2juHoTbVZR8MC3uexqft68o1JJdaagoCbOiKLKPjAeMpV1Q3qKuL6i/rJpb2sY5wTD3CpBnRzRDPP7+RN5tby4uCHZ8MrRkfwpl8g7FDpxcObA1w1/V1/3cTPGR6RWg4ppjntlOOI07oQlnUOE9RDy5XGUrim8LmN0/QPwTAAD//yu9sjUQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", "snippets": null, "strict": false }, @@ -264,273 +264,8 @@ "verify": true }, "sensitive_attributes": [], - "identity_schema_version": 0 - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "control_plane1_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "control-plane1-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/control-plane1-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBSRZehrOGIg0ZyIHyC79riKuanvWr6Sqv68eCAFnCxhllNMCEn4VHs/77gCgQCo3qcEGOVuDXo9uNnR0Wo3ko1myUmv8i1+ltO7gxg+L0tvbf0QbBmfDLm4BAAD//4CQfvX4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/opt/bin/kubeadm reset -f\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create ceph namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\n\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStart=/usr/bin/systemctl disable install-ceph.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n }\n ]\n }\n}", - "file_name": "control-plane1-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.control_plane1_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "control_plane2_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "control-plane2-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/control-plane2-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuFMBRG9/yUDjE3FcVABsFSOlSkHcUhJLco1aQkUeu/f+h76PC407mH77SfKuq+I7WaUGLsXwhpa4yr878dKY3xGIL8MYyJ/bgQBU+y9DScMRBpzkQOkF36XUVc1fasX0lVf189EALOFjDKKKcFT/hVeDzvuwOAAqncpAYb5GwNej262dDRaTWSj2bJSq3xL36V0rqDGz8sSm9v/xFtGJwNu7gFAAD//0NI5lj4AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwWobQQyG7/sUOjoBj68lN5MGWmgTaJJDe5M1ylp4drRImoXt0xd7c+jBNYX4v8xISB/fMLcfT3cL/8j29eXL04+vvx4+w/b+/uH5GZ4ev/08M3iB8d85MV724uCzBw8gDiNagL4BgjM14wxS3ww9rFE043SOsS0FkEImCWEHNIZBq4Qe97FmKNr3nNMlj9eKLfZq8vu4Q8TuoAaDeHM+mnmYUJQZRtO97CTe2X8xBpzB2FsJkApZnGQsUtHmxYJ7XES1pvMeV/jbbn3ldIe2Y6sc7EBaw7QwjAUrw6Nm7rrvWLFnmATvTrMUBVbvl5uOsAgprJbz1Jq4sCmsYYd0aCOslsZNh9YrZdhHjH632Szl2tkmtrRUySdKh0+eWs1sVLTlVJSwbK7/8D8BAAD//2cEMuhrAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSv27bPBTFdz4FofmTRFmy5XDzl2RoAxRGXaRD0YEir2xCMq/AP0qcpy8oqakddCu8GOI55L3nd8Sgn8E6jYbTLjQg1Dnrti7TmI9FA16UpNNGcfoZtblH0+pjsMJrNESi8Rb7fS8McEJpj1L0u/2nR6MG1MbHb5QKNYL12sFOKQvOcZq0ijEefyvO71bJJGu0UXu0ntNNVZWEUhldrZbCwxNcogugliWsZSuFrKGq60puoJWCVZtKVCu1vavbkrV1yVhZMbVZ19tmLdumEqUqqyYhBhV8haN2ft4gDmjEGThdVkmHuMsqvm71AWUHntNg9CvP89wGk0ed0AasuvqbOZQdoVN8PfjHV2/Fzh7dvH98NNUDp8mKsYJXNeN1UWyud5d9cB5sqkxMp2AZy+6KbF3+d+toecH5upw9I/bhDOnQh6M2qdKW0yTHwee9buAVZB6nsQY8uHwZbBG7fPbmkywhSjuJI9hLnLdB9DGe4Rt2YBaCgz6AHcG+g6XJj2uGBWM/ecQ2j+YnK0268VicZOZLCy842sFVq43FdhYF40QLh04P97tnsDPpqYXeBiBpmpKP1ezBZ3Jq4G1Di6WhT7PmtqTivXWcJ+QEovent/+1UbubgwXBA56FnmbfuiwYBVb2GFQ2VfuP6sshJpN+RHoFKL3mmBB5tBiGB6tHiKDcxXk4q4SSGcV+AvPwDxBF8Ccw/neIhFJh0FzOGJYaghFND2qJl9IXaE6I3d8O411o9dv7VWdUwOn32UF+BQAA//8OJfhXMgQAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubeadm.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "control-plane2-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.control_plane2_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "control_plane3_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "control-plane3-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/control-plane3-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMMWuHMBBH93yUDjEXRTGQQbCUDhVpR3EIyRWlmpQkav32RVt0+HPTu8fvdW8q6qEnjZpRYhyeCOkajJvzXz2pjPEYgvw0jInjuBBlmuTZZThjILKCiQIgv/WLirip/VGnpG4+7h4IAVcLGGWU0zJN+F34f/7tTgAKpHazGm2QizXo9eQWQyen1URe2zWvtMbv+F5J605u/bgqvT//RLRhdDYc4jcAAP//AgBuPPgAAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSP2/bMBDFd34KQnMlUZZsOdzcJEMboDDqIh2KDhR5sgnJPIF/lDifvqCkpnFQdCm8GOI78t37PTHoR7BOo+G0Cw0Idc66rcs05mPRgBcl6bRRnH5GbW7RtPoYrPAaDZFovMV+3wsDnFDaoxT9bv/p3qgBtfHxG6VCjWC9drBTyoJznCYrxgpe1YzXRbHhK85vymTSNtqoPVrP6aaqSkKpjKOtlsLDA1w4TVqAWpawlq0Usoaqriu5gVYKVm0qUa3U9qZuS9bWJWNlxdRmXW+btWybSpSqrJqEGFTwFY7a+XmN6NKIM3C67JMOcaHpdasPKDvwnAajn3me5zaYPOqENmDVm7+ZQ9kROmXYg79/9lbs7NHNIcRHUz38a3fZB+fBpsrEiAqWseymyNblh+uJlhecr5eZEftwhnTow1GbVGnLaZLj4PNeN/AMMo9urAEPLl+MLWKXz7P5JEuI0k7iCPYS/TaIPsYzfMMOzIJx0AewI9hXujT50SrGePytOC8Y+8kjttman0Zp0o3H4iQzX1p4wtEOrlptLLazKBgnWjh0erjdPYKdSU9V9DYASdOUvO9nDz6TUw2va1osNX2YNddNFa/V4zwhJxC9P7181Ebtrg4WBHd4FnryvnVZMAqs7DGobOr3H9WXQ0wmfY/0DaD0LceEyKPFMNxZPUIE5S7Ow1kllMwo9hOYu/+AKII/gfG/QySUCoPmcsaw1BCMaHpQS7yUPkFzQuz+dhjvQqtfXq86owJOv88T5FcAAAD//xliG4M3BAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "control-plane3-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.control_plane3_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "worker1_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "worker1-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/worker1-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASAR/z7FQCAGVWgCwQ8xv/ohOterv3T6yqP64iSolnDIEDF3xPiCvx//0bHoAcWeUnPbioZmcpmNHPlo/e6JG9NkteGkPf6b1Uzh/chGHRZnv+SeTi4F3cxW8AAAD//9kKqIr7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5yTbW6EIBBA/3MKLrBkPpo1ndtYtGmzRAzdTa/fYNEopaU4/sLhvcSXiNQZMGBQa+287d2b/7grkXj+nu2tfp+vl/zk55fe3pQiAJSnDqRDvAqJPKPW1k/34N1ldv00onlMwxis84/BLJpsX3BQ5qCKgwoOzhxccfBPBwLqTx9uYyh8RFqUKDpFcVrybxQr9ToASHxS6TiNtQ8KKijqsQ8KLijqrfeKmDlONdoRojMQ76C/QmP8OcgsjRsDryi1h11Rbg+a0KVltcl2mVou8z/CfQUAAP//Msgv7lgEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRz27bPBDE73oKQudPf2g7Tr69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+dnBnpFCcQeVJeOaNy57J5CSVyN+ojRrLOOvAP1mck/s2+oTWIisc88O/yOLYW4HECmlDdnBPXG0qHoTCkrdGDbYQSVPF2gqipJvrLsoyGP4m62ZWDbZWqO0WN8uUQxO2nD9K5S028FDaDyVV1r2DzW8Kj1FlYAutb5rLF9ChGlcD6AynVd1uX/unxY/3dvaUADPKwXz8h9OmMx9KklXzgSUHnFQ6x6OuIFbTXFEY8RQ3VNdhWHavFWsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV9cwrXmQ+hdsN5trtDhbVd6NrT7ZMq4F33iUIWxWW+FmESUfTIOHjobn3SsKNWQXGipKwqwoiuwj4x5jaWeU96j1FfWXRXNP2zgnGKZeAfLshKaPp9+fyLvd3cUVwZ7PhubsT6FM3qHYnpMre7am/6v6epiaKT4yvQFU3HLMM9sKp2EvNOIEKryHiGeXq2xB8W0Gs/8HiH8CAAD//1pu0RwQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "worker1-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.worker1_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "worker2_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "worker2-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/worker2-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASQTzm2akEAMqsAFkg5jf+RSda9Xbvn1hVf1xFlBLPGAIHLvieEFfi//s3PAA5sspPenBRzc5SMKOfLR+90SN7bZa8NIa+03upnD+4CcOizfb8k8jFwbu4i98AAAD//8ZKOy/7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRwW7bPBCE73oKQudfomQ7Tv69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+czA72iBGIPqktHNO5cdk+hJNZjfcRo1llH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VBWmVJW6MC2wwgqebqA1lqS15Z9NORR3M22DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVS+qqoaNo8VPNb1FlYAdbXKZ43tU4gohfMBVF5XZVX+X5cP6//uLQ3UAA/rxTNyn85YDH1qyReOBFSueYi6pyNe0OopjniMGPQ12VUc9OLVsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV1UwrXmQ6hdsN5trtDhbVd6NbX2yZVwLvvEoQ9istsLNIko+mAYPHQ3Pu1cUasguNFSUhFlRFNlHxj3G0s4o71HXV9RfFs09beOcYJh6BcizE5o+nn5/Iu92dxdXBHs+G5qzP4UyeYdie06u7Nma/q/q62FqpvjI9AZQccsxz2wrnIa90IgTqPAeIp5drrIFxbcZzP4fIP4JAAD//3XfDwQQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "worker2-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.worker2_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "worker3_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "worker3-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/worker3-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLd/KIQyEGwlB4q0h7FQ0i2KNWkJFHr2xdt0cLPnmaHb9pXnUzfsVpPpCj1D4y1NaXVh8+OldYGilF9WAC5n5AS4faYZ6cSACizAmSBmP/zzzrRqrd7f2NV/X4VUUo8YwgcuOB7QlyJv+/v8ADkyCo/6cFFNTtLwYx+tnz0Ro/spVny0hj6Sm+lcv7gJgyLNtvTdyIXB+/iLn4CAAD//zN1Skz7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5ST7W6DIBRA//MUvEDJ/Vhqdt/GqcuWEjGszV5/wVEjlBa5/sLLOYknEakzYMCg1tq6obdf7ueqRML5f7a3+ns5n/KTWz764aIUAaC8dSAd4llI5B21Htx89c6eFtvPE5rbPE5+sO42mlWT7QsOyhxUcVDBwZmDKw5+dCCg/nX+MvnCR8RFiaK4pGcUlSiOS35GsVKfI4CEJ5YO01g7UVBBUY+dKLigqLfeK0LmMNXUKUQ76EXpFOId9Co0hp+DzNq4MfAdpfawd5Tbg0Z0bVkNuV2mAwG3y3wg3F8AAAD//+yy88FYBAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRPY/bPBCEe/0KQvUrUTr7Pt7tnPM1CRAEcXBNkIImV/JCMldYkjpffn2gDyD2tQEbgpwhZ+cxA72iBGIPqktHNO5cdk+hJNZjfcRoNllH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VA2mVJW6MC2wwgqebqA1lqS15Z9NORR3NW2DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVR+V1U1bB8reKzrB7gDqKtNPmtsn0JEKZwPoPK6Kqvy/7q83/x3a2mgBrhfPSP36YzF0KeWfOFIQOWah6h7OuIFrZ7iiMeIQa/JVnHQi1fPsjxzFCyPKO9T4CNznIoZfnCHfhnBDHRAGVFevBuYfASV/2xcVcG05kGqX/Cw3a7R4mxVeTe29cmWcSP4xqMMYXv3INwsouSDafDQ0fC8e0WhhuxCQ0VJmBVFkX1k3GMs7YzyFnW9ov6yaG5pG+cEw9QrQJ6d0PTx9PsTebe7uVgR7PlsaM7+FMrkHYrtObmyZ2v6v6qvh6mZ4iPTK0DFNcc8s61wGvZCI06gwnuIeHa5yhYU32Yw+3+A+CcAAP//r02VuhADAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "worker3-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.worker3_ignition" - ] - } - ] - }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_file", - "name": "worker4_ignition", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content_type": "snippets", - "datastore_id": "cephfs", - "file_mode": null, - "file_modification_date": null, - "file_name": "worker4-ignition-user-data", - "file_size": null, - "file_tag": null, - "id": "cephfs:snippets/worker4-ignition-user-data", - "node_name": "hyper1", - "overwrite": true, - "source_file": [], - "source_raw": [ - { - "data": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker4%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SMwWqGMBAG73mUHtLdVBQCOQiW0kNF2qN4CMkWpZqUJGp9+6It+sPPnmaHb9o3nUzfsVpPpCj1D4y1NaXVh6+OldYGilF9WgC5n5ASIXvMs1MJAJRZAbJAzG/8i0606u3eP7Gq/riKKCWeMQQOXPA9Ia7E//dveAByZJWf9OCimp2lYEY/Wz56o0f22ix5aQx9p/dSOX9wE4ZFm+35J5GLg3dxF78BAAD//7nMbL/7AAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zQX27DIAwG8PeeIhcIyqos6h56FuQS01kFGxmWtrefsmXZHzE13wsS+iH7g7EYSlNvKFkvegUdm785Nk+7xQ2bnRP2BkIwiyY+V1y/xZ2UxjMuR8u+dRBCS6nAKWB+7IYFfjnG4ikUVMPeOmEuCu5iI9zWud1HKltqsp9vf7TuKq1HyvNQO99+uykamVCdxEjFRoyi99ov+myIpZC/mwg3+5ZR7RWKe53rru553+8PhyomzgXYrfzY9N3LsLugMgaTgMk1/2Ru/wtaYSuScm3R9wAAAP//X0LFgj8CAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRv47bPBDEez0FofqTKJ11f77tnPM1CRAEcXBNkIImV/JCMldYkjpfnj6QZCD2tQEbgpwhZ+dnRnpFCcQeVJ8OaNyp7J9CSayn+oDRbLKevAP1mck/s2+pS2Iisc88O/yOHYW4HkCmlDcnBPXG0qM0mVJWaM+2xwgqeTqD1lqS15Z9NORR3NW2DGz7TC0xBowv5yhmK12Y31Vq/q2gEVR+V1U1NI8VPNb1A9wB1FWTLxo7pBBRCucDqLyuyqr8vy7vN//dWlqoAe43q2fiIZ2wGIfUkS8cCahc8xj1QAc8o9VzHPEYMehLsos46NWrF1meOQqWJ5T3OfCBOc7FjD+4R7+OYEbao0woL96NTD6Cyn+2rqpgXssg1S94aJpLtLhYVd5PXX20ZdwIvvEkY2juHoTbVZR8MC3uexqft68o1JJdaagoCbOiKLKPjAeMpV1Q3qKuL6i/rJpb2sY5wTD3CpBnRzRDPP7+RN5tby4uCHZ8MrRkfwpl8g7FDpxcObA1w1/V1/3cTPGR6RWg4ppjntlOOI07oQlnUOE9RDy5XGUrim8LmN0/QPwTAAD//yu9sjUQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n }\n ]\n }\n}", - "file_name": "worker4-ignition-user-data", - "resize": 0 - } - ], - "timeout_upload": 1800 - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.worker4_ignition" - ] + "private": "eyJvcmlnaW5hbF9zdGF0ZV9zaXplIjoiTlRjek1qUXpNemt5In0=" } ] }, @@ -566,7 +301,7 @@ "affinity": "", "architecture": "", "cores": 4, - "flags": null, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -601,7 +336,17 @@ ], [ "10.0.2.91" - ] + ], + [], + [], + [ + "10.0.10.0" + ], + [], + [], + [], + [], + [] ], "ipv6_addresses": [ [ @@ -610,20 +355,50 @@ [ "2001:470:7116:2::91", "fd00:0:0:2::91", - "fe80::be24:11ff:fe96:e604" + "fe80::be24:11ff:feb2:a725" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:8495:c8a0:d59b:d0c0" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:96:E6:04" + "BC:24:11:B2:A7:25", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:78:AC:F8:81:98", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" ], "machine": null, "memory": [ { - "dedicated": 4096, - "floating": 4096, + "dedicated": 8192, + "floating": 8192, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -637,7 +412,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:96:E6:04", + "mac_address": "BC:24:11:B2:A7:25", "model": "virtio", "mtu": 0, "queues": 0, @@ -648,7 +423,15 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "calieae95b74f2c", + "cali7c65858f1d3", + "tunl0", + "vxlan-v6.calico", + "caliddba80e9b5e", + "cali58a55eebea1", + "calife47df6812f", + "calicb53b7a002e" ], "node_name": "hyper1", "numa": [], @@ -738,7 +521,7 @@ "affinity": "", "architecture": "", "cores": 4, - "flags": null, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -773,7 +556,14 @@ ], [ "10.0.2.92" - ] + ], + [], + [ + "10.0.10.64" + ], + [], + [], + [] ], "ipv6_addresses": [ [ @@ -782,20 +572,38 @@ [ "2001:470:7116:2::92", "fd00:0:0:2::92", - "fe80::be24:11ff:feaf:ca44" + "fe80::be24:11ff:fee4:d5d6" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:63eb:d7a8:d9b3:1e00" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:AF:CA:44" + "BC:24:11:E4:D5:D6", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:42:B5:2A:6A:1B", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" ], "machine": null, "memory": [ { - "dedicated": 3072, - "floating": 3072, + "dedicated": 8192, + "floating": 8192, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -809,7 +617,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:AF:CA:44", + "mac_address": "BC:24:11:E4:D5:D6", "model": "virtio", "mtu": 0, "queues": 0, @@ -820,7 +628,12 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "calia2e2dfb30a3", + "tunl0", + "vxlan-v6.calico", + "cali25103b133f8", + "calib7a33493c64" ], "node_name": "hyper2", "numa": [], @@ -914,7 +727,7 @@ "affinity": "", "architecture": "", "cores": 4, - "flags": null, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -949,7 +762,13 @@ ], [ "10.0.2.93" - ] + ], + [], + [], + [ + "10.0.10.192" + ], + [] ], "ipv6_addresses": [ [ @@ -958,20 +777,34 @@ [ "2001:470:7116:2::93", "fd00:0:0:2::93", - "fe80::be24:11ff:fe88:ad6a" + "fe80::be24:11ff:fe69:4638" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:edec:9c79:57a2:840" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:88:AD:6A" + "BC:24:11:69:46:38", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:EE:86:C9:B8:69" ], "machine": null, "memory": [ { - "dedicated": 3072, - "floating": 3072, + "dedicated": 8192, + "floating": 8192, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -985,7 +818,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:88:AD:6A", + "mac_address": "BC:24:11:69:46:38", "model": "virtio", "mtu": 0, "queues": 0, @@ -996,7 +829,11 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "cali6ae1ad1f505", + "calicfb3e89fc4e", + "tunl0", + "vxlan-v6.calico" ], "node_name": "hyper3", "numa": [], @@ -1090,7 +927,7 @@ "affinity": "", "architecture": "", "cores": 1, - "flags": null, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -1112,7 +949,7 @@ "import_from": "cephfs:import/flatcar_production_proxmoxve_image.qcow2", "interface": "virtio0", "iothread": false, - "path_in_datastore": "vm-999-disk-0", + "path_in_datastore": "base-999-disk-0", "replicate": true, "serial": "", "size": 50, @@ -1143,7 +980,7 @@ "keyboard_layout": "en-us", "kvm_arguments": "", "mac_addresses": [ - "BC:24:11:C0:A0:19" + "BC:24:11:8D:8B:89" ], "machine": "", "memory": [ @@ -1163,7 +1000,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:C0:A0:19", + "mac_address": "BC:24:11:8D:8B:89", "model": "virtio", "mtu": 0, "queues": 0, @@ -1250,8 +1087,8 @@ { "affinity": "", "architecture": "", - "cores": 2, - "flags": null, + "cores": 14, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -1286,7 +1123,21 @@ ], [ "10.0.2.101" - ] + ], + [ + "10.0.10.103" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] ], "ipv6_addresses": [ [ @@ -1295,20 +1146,66 @@ [ "2001:470:7116:2::101", "fd00:0:0:2::101", - "fe80::be24:11ff:fe7b:c7f9" + "fe80::be24:11ff:fe97:6379" + ], + [], + [ + "2001:470:7116:a:4e89:604c:ea1a:a982" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:7B:C7:F9" + "BC:24:11:97:63:79", + "00:00:00:00:00:00", + "66:ED:A6:2D:2B:8D", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" ], "machine": null, "memory": [ { - "dedicated": 10240, - "floating": 10240, + "dedicated": 25000, + "floating": 25000, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -1322,7 +1219,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:7B:C7:F9", + "mac_address": "BC:24:11:97:63:79", "model": "virtio", "mtu": 0, "queues": 0, @@ -1333,9 +1230,21 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "tunl0", + "vxlan-v6.calico", + "cali438de1772bb", + "cali150b6f155d8", + "cali61a9554ce92", + "cali97e829f5c20", + "cali71eae69175a", + "cali482326879bd", + "cali471d98baa74", + "cali19f1a84797b", + "cali4c30009dc88", + "cali2fc5f4481bb" ], - "node_name": "hyper1", + "node_name": "hyper4", "numa": [], "on_boot": true, "operating_system": [], @@ -1434,8 +1343,8 @@ { "affinity": "", "architecture": "", - "cores": 2, - "flags": null, + "cores": 6, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -1470,7 +1379,50 @@ ], [ "10.0.2.102" - ] + ], + [], + [ + "10.0.10.128" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] ], "ipv6_addresses": [ [ @@ -1479,20 +1431,182 @@ [ "2001:470:7116:2::102", "fd00:0:0:2::102", - "fe80::be24:11ff:fec5:1c22" + "fe80::be24:11ff:fec2:408a" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:2903:60e7:598f:e9c0" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:C5:1C:22" + "BC:24:11:C2:40:8A", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:65:83:E5:BB:CC", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" ], "machine": null, "memory": [ { - "dedicated": 10240, - "floating": 10240, + "dedicated": 14000, + "floating": 14000, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -1506,7 +1620,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:C5:1C:22", + "mac_address": "BC:24:11:C2:40:8A", "model": "virtio", "mtu": 0, "queues": 0, @@ -1517,9 +1631,50 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "calibce684fc6dd", + "tunl0", + "vxlan-v6.calico", + "calic5dffe1f987", + "cali9aa1fde2946", + "cali8512da79659", + "cali84cad734eaa", + "caliaa9e304886c", + "calidc5d61e997c", + "calidd602456e41", + "cali362ed739e39", + "cali17dc2dabdfa", + "cali8d2065ea893", + "calibb76b3ab372", + "cali933e1efdc43", + "cali2e499c0605e", + "cali46a83cb5b38", + "cali4b44079d8c4", + "cali8d3cee2bc69", + "cali2a4eb726f83", + "caliadd23907588", + "cali2ffe1c36b3b", + "cali0816a005e97", + "caliba9d140bc58", + "calic9eae6671be", + "califd0c1ceda99", + "cali410d4ce3d0a", + "cali119fc794523", + "cali815f9f28e57", + "cali561dcc29b32", + "cali3ef69860863", + "cali87a86cd44ac", + "cali22b64eb4620", + "calic15955a29e3", + "calie94c3337bf8", + "cali272f9b8fb79", + "calic43f18bdbfe", + "cali21e0604c323", + "cali48a28da992f", + "calia5cad583db5", + "calibef07565dae" ], - "node_name": "hyper2", + "node_name": "hyper6", "numa": [], "on_boot": true, "operating_system": [], @@ -1587,190 +1742,6 @@ } ] }, - { - "mode": "managed", - "type": "proxmox_virtual_environment_vm", - "name": "worker3", - "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acpi": true, - "agent": [], - "amd_sev": [], - "audio_device": [], - "bios": "seabios", - "boot_order": [ - "virtio0" - ], - "cdrom": [], - "clone": [ - { - "datastore_id": "", - "full": true, - "node_name": "hyper1", - "retries": 1, - "vm_id": 999 - } - ], - "cpu": [ - { - "affinity": "", - "architecture": "", - "cores": 2, - "flags": null, - "hotplugged": 0, - "limit": 0, - "numa": false, - "sockets": 1, - "type": "host", - "units": 1024 - } - ], - "description": "kubernetes worker3", - "disk": [], - "efi_disk": [], - "hook_script_file_id": null, - "hostpci": [], - "id": "1013", - "initialization": [ - { - "datastore_id": "Pool1", - "dns": [], - "interface": "ide2", - "ip_config": [], - "meta_data_file_id": "", - "network_data_file_id": "", - "type": "", - "user_account": [], - "user_data_file_id": "cephfs:snippets/worker3-ignition-user-data", - "vendor_data_file_id": "" - } - ], - "ipv4_addresses": [ - [ - "127.0.0.1" - ], - [ - "10.0.2.103" - ] - ], - "ipv6_addresses": [ - [ - "::1" - ], - [ - "2001:470:7116:2::103", - "fd00:0:0:2::103", - "fe80::be24:11ff:fec1:ff6" - ] - ], - "keyboard_layout": "en-us", - "kvm_arguments": null, - "mac_addresses": [ - "00:00:00:00:00:00", - "BC:24:11:C1:0F:F6" - ], - "machine": null, - "memory": [ - { - "dedicated": 10240, - "floating": 10240, - "hugepages": "", - "keep_hugepages": false, - "shared": 0 - } - ], - "migrate": false, - "name": "worker3", - "network_device": [ - { - "bridge": "vmbr0", - "disconnected": false, - "enabled": true, - "firewall": false, - "mac_address": "BC:24:11:C1:0F:F6", - "model": "virtio", - "mtu": 0, - "queues": 0, - "rate_limit": 0, - "trunks": "", - "vlan_id": 0 - } - ], - "network_interface_names": [ - "lo", - "eth0" - ], - "node_name": "hyper3", - "numa": [], - "on_boot": true, - "operating_system": [], - "pool_id": null, - "protection": false, - "reboot": false, - "reboot_after_update": true, - "rng": [], - "scsi_hardware": "virtio-scsi-pci", - "serial_device": [], - "smbios": [], - "started": true, - "startup": [ - { - "down_delay": -1, - "order": 1, - "up_delay": -1 - } - ], - "stop_on_destroy": false, - "tablet_device": true, - "tags": [ - "flatcar", - "kubernetes", - "terraform", - "worker" - ], - "template": false, - "timeout_clone": 1800, - "timeout_create": 1800, - "timeout_migrate": 1800, - "timeout_move_disk": 1800, - "timeout_reboot": 1800, - "timeout_shutdown_vm": 1800, - "timeout_start_vm": 1800, - "timeout_stop_vm": 300, - "tpm_state": [], - "usb": [], - "vga": [], - "virtiofs": [], - "vm_id": 1013, - "watchdog": [] - }, - "sensitive_attributes": [], - "identity_schema_version": 0, - "private": "bnVsbA==", - "dependencies": [ - "data.ct_config.control_plane1_ignition", - "data.ct_config.control_plane2_ignition", - "data.ct_config.control_plane3_ignition", - "data.ct_config.worker3_ignition", - "null_resource.wait_for_cluster_ready", - "null_resource.wait_for_cp1_api", - "null_resource.wait_for_cp2_cp3_api", - "proxmox_virtual_environment_download_file.flatcar_image", - "proxmox_virtual_environment_file.control_plane1_ignition", - "proxmox_virtual_environment_file.control_plane2_ignition", - "proxmox_virtual_environment_file.control_plane3_ignition", - "proxmox_virtual_environment_file.worker3_ignition", - "proxmox_virtual_environment_vm.control_plane1", - "proxmox_virtual_environment_vm.control_plane2", - "proxmox_virtual_environment_vm.control_plane3", - "proxmox_virtual_environment_vm.flatcar_template" - ] - } - ] - }, { "mode": "managed", "type": "proxmox_virtual_environment_vm", @@ -1802,8 +1773,8 @@ { "affinity": "", "architecture": "", - "cores": 2, - "flags": null, + "cores": 4, + "flags": [], "hotplugged": 0, "limit": 0, "numa": false, @@ -1838,7 +1809,43 @@ ], [ "10.0.2.104" - ] + ], + [ + "10.0.10.67" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] ], "ipv6_addresses": [ [ @@ -1847,20 +1854,154 @@ [ "2001:470:7116:2::104", "fd00:0:0:2::104", - "fe80::be24:11ff:feda:7268" + "fe80::be24:11ff:fe93:db44" + ], + [], + [ + "2001:470:7116:a:ed2a:1bf0:eec6:2100" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" ] ], "keyboard_layout": "en-us", "kvm_arguments": null, "mac_addresses": [ "00:00:00:00:00:00", - "BC:24:11:DA:72:68" + "BC:24:11:93:DB:44", + "00:00:00:00:00:00", + "66:41:F1:FA:94:3B", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" ], "machine": null, "memory": [ { - "dedicated": 12000, - "floating": 12000, + "dedicated": 14000, + "floating": 14000, "hugepages": "", "keep_hugepages": false, "shared": 0 @@ -1874,7 +2015,7 @@ "disconnected": false, "enabled": true, "firewall": false, - "mac_address": "BC:24:11:DA:72:68", + "mac_address": "BC:24:11:93:DB:44", "model": "virtio", "mtu": 0, "queues": 0, @@ -1885,7 +2026,41 @@ ], "network_interface_names": [ "lo", - "eth0" + "eth0", + "tunl0", + "vxlan-v6.calico", + "cali03cb18accdc", + "cali84ee2132997", + "calic8e313ad455", + "cali618b06a08bf", + "calie5d1061f0d2", + "cali74c2c7cab95", + "calid327fb542ec", + "cali57272c14d48", + "cali906311f4498", + "cali2ac9d3e29fd", + "cali6b5a4ca76d8", + "calid0c0262c20d", + "cali191b7b14a78", + "cali473e637c0db", + "calif655aa2ef63", + "cali84d3e8d6131", + "cali82198d56117", + "cali86df6a2a24e", + "califf0314f94ea", + "cali1299561ac78", + "cali12908b5e00f", + "cali482ab78782d", + "calid0b4ea119f4", + "cali17d2589e01b", + "calife77b2f6e7d", + "cali671375f849e", + "cali4c9d0a480a9", + "calicaaec793295", + "cali27e516903c7", + "cali820b67db9a6", + "calia6eb8e5554a", + "calif7837571d88" ], "node_name": "pbs", "numa": [], diff --git a/terraform/terraform.tfstate.backup b/terraform/terraform.tfstate.backup index 0e73edf..abcdf20 100644 --- a/terraform/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.backup @@ -1,9 +1,2135 @@ { "version": 4, - "terraform_version": "1.12.2", - "serial": 1573, + "terraform_version": "1.14.7", + "serial": 1608, "lineage": "d92c42be-29f9-bad9-ef9a-3dc952ff5fa5", "outputs": {}, - "resources": [], + "resources": [ + { + "mode": "data", + "type": "ct_config", + "name": "control_plane1_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 755\n - path: /etc/kubernetes/manifests\n #overwrite: true\n mode: 0755\n - path: /etc/install-calico\n overwrite: true\n mode: 0755\n - path: /var/lib/undercloud-stamps\n mode: 0755\n\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::91/64\n Address=2001:470:7116:2::91/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.91/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /opt/bin/velero\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\"\n\n - path: /etc/kubernetes/kubeadm-init.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n bootstrapTokens:\n - token: \"kvg1hc.t3rewovrps426rof\"\n description: \"default kubeadm bootstrap token\"\n ttl: \"0\"\n nodeRegistration:\n name: control-plane1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::91\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::91\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n controlPlaneEndpoint: \"[fd00:0:0:2::100]:6443\"\n networking:\n podSubnet: \"2001:470:7116:a::/64,10.0.10.0/24\"\n serviceSubnet: \"2001:470:7116:f:1::/108,10.0.91.0/24\"\n dnsDomain: \"k8s.undercloud.local\"\n controllerManager:\n extraArgs:\n flex-volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n bind-address: '::'\n apiServer:\n extraArgs:\n enable-aggregator-routing: \"true\"\n proxy-client-cert-file: /etc/kubernetes/pki/front-proxy-client.crt\n proxy-client-key-file: /etc/kubernetes/pki/front-proxy-client.key\n requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt\n requestheader-allowed-names: front-proxy-client\n requestheader-extra-headers-prefix: X-Remote-Extra-\n requestheader-group-headers: X-Remote-Group\n requestheader-username-headers: X-Remote-User\n\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n volumePluginDir: /opt/libexec/kubernetes/kubelet-plugins/volume/exec\n cgroupDriver: \"systemd\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\n - path: /etc/kubernetes/addons/kube-dns-fixed-svc.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: v1\n kind: Service\n metadata:\n name: kube-dns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n spec:\n type: ClusterIP\n ipFamilyPolicy: RequireDualStack\n ipFamilies: [IPv6, IPv4]\n clusterIP: 2001:470:7116:f:1::53\n clusterIPs:\n - 2001:470:7116:f:1::53\n - 10.0.91.53\n ports:\n - name: dns\n port: 53\n protocol: UDP\n targetPort: 53\n - name: dns-tcp\n port: 53\n protocol: TCP\n targetPort: 53\n - name: metrics\n port: 9153\n protocol: TCP\n targetPort: 9153\n selector:\n k8s-app: kube-dns\n\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"setting timezone to Europe/Berlin\"'\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-init.service\n enabled: true\n contents: |\n [Unit]\n Description=Kubeadm Init Cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n ConditionPathExists=!/etc/kubernetes/kubelet.conf\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/opt/bin/kubeadm reset -f\n\n ExecStart=/bin/sh -c 'echo \"kubeadm-init.service started...\"'\n\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"running kubeadm init...\"'\n ExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\n \n # copy files for kubectl\n ExecStart=/bin/sh -c 'echo \"copying files (admin.conf) to core home folder.\"'\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n\n - name: install-calico.service\n enabled: true\n contents: |\n [Unit]\n Wants=kubeadm-init.service\n After=kubeadm-init.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\n\n\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.calico.service started...\"'\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n ExecStart=/bin/sh -c 'echo \"witing 30s...\"'\n ExecStart=/bin/sleep 30s\n ExecStart=/bin/sh -c 'echo \"create calico namespace...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\n ExecStart=/bin/sh -c 'echo \"install tigera operator...\"'\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\n ExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 60s\n ExecStart=/bin/sh -c 'echo \"witing for tigera operator... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"create clico custom ressources...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\n \n ExecStart=/bin/sh -c 'echo \"witing 3m..\"'\n ExecStart=/bin/sleep 3m\n #ExecStart=/bin/sh -c 'echo \"apply calico (calico-apiserver)...\"'\n #ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\n #ExecStart=/bin/sh -c 'echo \"witing 1m...\"'\n #ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"witing calico-apiserver... (20mini max)\"'\n ExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sh -c 'echo \"witing 120s...\"'\n ExecStart=/bin/sleep 2m\n ExecStart=/bin/sh -c 'echo \"apply calico-peers...\"'\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\n ExecStart=/bin/sh -c 'echo \"witing 60s...\"'\n ExecStart=/bin/sleep 1m\n ExecStart=/bin/sh -c 'echo \"apply calico-ippools...\"'\n ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\n \n #ExecStart=/bin/sh -c 'echo \"witing for whisker..\"'\n #ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\n #ExecStart=/bin/sh -c 'echo \"port-forward -n calico-system service/whisker 8081:8081\"'\n #ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\n ExecStart=/usr/bin/systemctl disable install-calico.service\n #RemainAfterExit=true\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-ceph.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\n\n [Service]\n Type=oneshot\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n StandardOutput=journal+console\n StandardError=journal+console\n\n ExecStart=/bin/sh -c 'echo \"install.ceph.service started...\"'\n ExecStart=/bin/sleep 30\n\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\n\n # this must NOT be ignored\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\n\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\n\n # readiness gates (only stamp if these succeed)\n ExecStart=/opt/bin/kubectl -n ceph rollout status deploy/csi-cephfsplugin-provisioner --timeout=5m\n ExecStart=/opt/bin/kubectl -n ceph rollout status ds/csi-cephfsplugin --timeout=5m\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\n ExecStartPost=-/usr/bin/systemctl disable install-ceph.service\n\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n - name: install-gitea.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-ceph.service\n After=install-ceph.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\n \n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n \n ExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\n ExecStart=/bin/sleep 4m\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\n ExecStart=/bin/sleep 60s\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\n ExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\n ExecStart=/bin/sleep 3m\n ExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\n ExecStart=/usr/bin/systemctl disable install-gitea.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: install-argocd.service\n enabled: true\n contents: |\n [Unit]\n Wants=install-calico.service\n After=install-calico.service\n ConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin\n Type=oneshot\n\n ExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\n \n ExecStart=/bin/sleep 1m\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\n ExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\n\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\n\n ExecStart=/bin/sleep 10s\n ExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\n\n\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\n ##ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\n #ExecStart=/bin/sleep 10m\n #ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\n \n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\n ExecStart=/usr/bin/systemctl disable install-argocd.service\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: pin-service-ips.service\n enabled: true\n contents: |\n [Unit]\n Description=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\n After=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n Wants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\n ConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\n\n\n [Service]\n Type=oneshot\n StandardOutput=journal+console\n StandardError=journal+console\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=PATH=/usr/bin:/usr/sbin:/opt/bin\n ExecStart=/bin/sh -eu -c '\\\n echo \"[pin-service-ips] waiting for API...\" ; \\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \u003e/dev/null 2\u003e\u00261 \u0026\u0026 break; sleep 2; done ; \\\n echo \"[pin-service-ips] ensure namespaces exist...\" ; \\\n kubectl get ns kube-system \u003e/dev/null ; \\\n kubectl get ns argocd \u003e/dev/null 2\u003e\u00261 || kubectl create ns argocd ; \\\n kubectl get ns calico-system \u003e/dev/null ; \\\n echo \"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\" ; \\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\n echo \"[pin-service-ips] replace Services with fixed ClusterIPs...\" ; \\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\n echo \"[pin-service-ips] done.\" \\\n '\n\n ExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\n Restart=on-failure\n RestartSec=120s\n [Install]\n WantedBy=multi-user.target\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "2362473131", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 755\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/manifests\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/install-calico\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/var/lib/undercloud-stamps\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQWuEMBAF4Pv8jd56yGay4rKBHIQtpYeKtEfxEJIpSjUpSdT674u21ULJ6b2Pl6mfdTJtA6UeSFFq7wHqktLsw3sDhbWBYlRvlnO5PiHlFU95tovgHGV24fKCmB9896gTzXr572f4pb+fnuFWvh4NSon7CeSMM8GueBLZPv4pv3dbQIZw84PuXFSjsxRM70fLem90D0/VlBfG0Ed6KZTzW65CN2mzPHwmcrHzLq7wFQAA//9FuT+UDwEAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/velero\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/velero\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-init.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6xV3W7jNhO951MQvNmbUD+WYie8y7dZfFgsWgSb/gFFLyhyJBOiSXVIOfY+fUHJNqKugwZFbcCQoTlHhzPnjORgfgEMxjtB+7EBqXdZfxcy4/N92UCUFemN04J+diZ+9K413YgyGu9I430MEeXwk+/BBUEo5TSma0FZv+/KrcpihfDi9ziEerVG3zJCKaUagkIzxOmpTEMrRxvPj6cX3plshsRoBWUFI85r+AqdSRUTAaHUyR0IqryL6C0frHRQEkoVmmeveoiCjs4cRJ7nOLo81UnjAPWryyx41RM6ibAQPx0iygfsplNRmh7KzSAoWxVFKepNITZluRYrIe7LWaCyY4iAXLsgKCuLrMjuy+y2ulkiWlEKcVvNmL234w74YMfOOK4NCspyP8TcmgYOoPKkBh1ECPlJ2Kk45DM2n8oYsV5J+/D0+ZPTgzcuJtlS7wGjCfCgNUIIb6tvjNNPHqOg67quiEqw1igZ4QscBWUtwEZVcKtaJdUG6s2mVmtolSzqdS3rlb6737RV0W6qoqjqQq9vN3fNrWqbWla6qhtGOOfk3Vb7OLdy6bbTdJ/ScC+npOz3VheFSN+VEGVR/CHSERhxEF889sZ1qRWD189j45IT/tYDKUS+rm+meaWffFWnjgTAvVHwBmgaYl4WdzfnOZ9x2oVHv5NmisBdyEanAZX1o86mEbHzOSzgD9LJDjDpg6XdWgsH/t+ZI1GmEXN5NsIHIT6keTwD7q8qACcbC1x2HUIno0eOfoypm5RFHGEmHdAfjlxZAy7y5BreGguC5hAX8obe5C16F/lrRKYwfk/Tw/HE8l6aHo4TDcKfI4S4BakBL6rkezXJi54lkbTWv4DmackEQb8XcAUzdZPPfwIfEFpzEPQ3/hV2PgKftgu/guvQj8MZJ+j8ucD+n+5eQY0BMKlbAC+onwMguRpACzFTU8iWOSxPOfwy1yxzePEQE4KRLUgbt9/+Z5x+WNw4rcN/CsOp6sfn+e3B3lyW/PVOZWQ2+NNk98cUjX+RDKKmdj+iSRmgLBxDhJ1mRI5xCy6mBXh6wUjn3XHnx0U6tKApCoTSF2i23vfXbiYuj+bbhWrnNQj664wgfwUAAP//F9AABn8HAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/addons/kube-dns-fixed-svc.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4yRwU7rMBBF9/6K+YCXKn6Uls62FVJ3FgU2iMXUGZAVJzb2JFL+HoWmEBYV9fLOuUejMUX3zCm70CL0WtWurRAOnHpnWTUsVJEQKoCWGkaouyMXVZunIEey5zQPWbhRAJ6O7PPYAajvckExzoo5sh1nMkRG2PouC6e9UQAu3lPj/GCCd3ZAeOCPziXedeQPQrb+QRxnhJe96Vf/YG/65asCsGcTwv+y1Lhcl7jWeoVvqBFvb+bItFxxkRxnulyUi41efAUxJPlunU5xusL4xiHCVASIKUiwwSM87cyUCaV3FjMHZ55CbPzL9bi9wtWwJGd/77XRV9smNLNnKyFd+sLPAAAA//8MLaPANAIAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\n\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"setting timezone to Europe/Berlin\\\"'\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Kubeadm Init Cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\nConditionPathExists=!/etc/kubernetes/kubelet.conf\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/opt/bin/kubeadm reset -f\\n\\nExecStart=/bin/sh -c 'echo \\\"kubeadm-init.service started...\\\"'\\n\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"running kubeadm init...\\\"'\\nExecStart=/opt/bin/kubeadm init --upload-certs --config=/etc/kubernetes/kubeadm-init.yaml\\n\\n# copy files for kubectl\\nExecStart=/bin/sh -c 'echo \\\"copying files (admin.conf) to core home folder.\\\"'\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\nExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-init.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=kubeadm-init.service\\nAfter=kubeadm-init.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-calico.done\\n\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.calico.service started...\\\"'\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nExecStart=/bin/sh -c 'echo \\\"witing 30s...\\\"'\\nExecStart=/bin/sleep 30s\\nExecStart=/bin/sh -c 'echo \\\"create calico namespace...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/namespace.yaml\\nExecStart=/bin/sh -c 'echo \\\"install tigera operator...\\\"'\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/operator-crds.yaml\\nExecStart=-/opt/bin/kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/tigera-operator.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 60s\\nExecStart=/bin/sh -c 'echo \\\"witing for tigera operator... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n tigera-operator tigera-operator --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"create clico custom ressources...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/custom-resources.yaml\\n\\nExecStart=/bin/sh -c 'echo \\\"witing 3m..\\\"'\\nExecStart=/bin/sleep 3m\\n#ExecStart=/bin/sh -c 'echo \\\"apply calico (calico-apiserver)...\\\"'\\n#ExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico.yaml\\n#ExecStart=/bin/sh -c 'echo \\\"witing 1m...\\\"'\\n#ExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"witing calico-apiserver... (20mini max)\\\"'\\nExecStart=/opt/bin/kubectl wait deployment -n calico-apiserver calico-apiserver --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sh -c 'echo \\\"witing 120s...\\\"'\\nExecStart=/bin/sleep 2m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-peers...\\\"'\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/calico-peer.yaml\\nExecStart=/bin/sh -c 'echo \\\"witing 60s...\\\"'\\nExecStart=/bin/sleep 1m\\nExecStart=/bin/sh -c 'echo \\\"apply calico-ippools...\\\"'\\nExecStart=-/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/ippools.yaml\\n\\n#ExecStart=/bin/sh -c 'echo \\\"witing for whisker..\\\"'\\n#ExecStart=/opt/bin/kubectl wait deployment -n calico-system whisker --for condition=Available=True --timeout=1200s\\n#ExecStart=/bin/sh -c 'echo \\\"port-forward -n calico-system service/whisker 8081:8081\\\"'\\n#ExecStart=/opt/bin/kubectl port-forward -n calico-system service/whisker 8081:8081\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-calico.done\\nExecStart=/usr/bin/systemctl disable install-calico.service\\n#RemainAfterExit=true\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-calico.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-ceph.done\\n\\n[Service]\\nType=oneshot\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nExecStart=/bin/sh -c 'echo \\\"install.ceph.service started...\\\"'\\nExecStart=/bin/sleep 30\\n\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-nodeplugin-rbac.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-provisioner-rbac.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/secrets.yaml\\n\\n# this must NOT be ignored\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/ceph-conf.yaml\\n\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin-provisioner.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-cephfsplugin.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-config-map.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csi-encryption-kms-config.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/csidriver.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/ceph/storage-class.yaml\\n\\n# readiness gates (only stamp if these succeed)\\nExecStart=/opt/bin/kubectl -n ceph rollout status deploy/csi-cephfsplugin-provisioner --timeout=5m\\nExecStart=/opt/bin/kubectl -n ceph rollout status ds/csi-cephfsplugin --timeout=5m\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-ceph.done\\nExecStartPost=-/usr/bin/systemctl disable install-ceph.service\\n\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-ceph.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-ceph.service\\nAfter=install-ceph.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-gitea.done\\n\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n ceph csi-cephfsplugin-provisioner --for condition=Available=True --timeout=1200s\\nExecStart=/bin/sleep 4m\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/secrets.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/db.yaml\\nExecStart=/bin/sleep 60s\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/adminer.yaml\\nExecStart=/opt/bin/kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/gitea/gitea.yaml\\nExecStart=/bin/sleep 3m\\nExecStart=/opt/bin/kubectl exec deploy/gitea -n gitea -- /bin/startup.sh\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-gitea.done\\nExecStart=/usr/bin/systemctl disable install-gitea.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-gitea.service\"\n },\n {\n \"contents\": \"[Unit]\\nWants=install-calico.service\\nAfter=install-calico.service\\nConditionPathExists=!/var/lib/undercloud-stamps/install-argocd.done\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\n\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin\\nType=oneshot\\n\\nExecStart=/opt/bin/kubectl wait deployment -n kube-system coredns --for condition=Available=True --timeout=600s\\n\\nExecStart=/bin/sleep 1m\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/namespace.yaml\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/install.yaml\\nExecStart=/opt/bin/kubectl wait deployment -n argocd argocd-server --for condition=Available=True --timeout=600s\\n\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/repo.yaml\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/apps.yaml\\n\\nExecStart=/bin/sleep 10s\\nExecStart=/opt/bin/kubectl apply -n argocd -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/ingress.yaml\\n\\n\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-bootstrap.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/repos/k8aux-apps.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/argocd.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/calico.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/rook-ceph.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/apps/gitea.yaml\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://aux-balancer.undercloud.cf:3000/undercloud/k8aux-bootstrap/raw/branch/main/argocd/argocd-secret.yaml\\n##ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl wait deployment -n gitea gitea --for condition=Available=True --timeout=4800s\\n#ExecStart=/bin/sleep 10m\\n#ExecStart=/opt/bin/kubectl apply -n argocd -f http://gitea.gitea.svc.k8aux.undercloud.cf:3000/undercloud/k8aux-apps/raw/branch/main/app-of-apps/app-of-apps.yaml\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/install-argocd.done\\nExecStart=/usr/bin/systemctl disable install-argocd.service\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"install-argocd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Pin fixed dual-stack ClusterIPs for kube-dns, argocd-server and whisker\\nAfter=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nWants=install-argocd.service install-calico.service kubeadm-init.service network-online.target\\nConditionPathExists=!/var/lib/undercloud-stamps/pin-service-ips.done\\n\\n\\n[Service]\\nType=oneshot\\nStandardOutput=journal+console\\nStandardError=journal+console\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=PATH=/usr/bin:/usr/sbin:/opt/bin\\nExecStart=/bin/sh -eu -c '\\\\\\n echo \\\"[pin-service-ips] waiting for API...\\\" ; \\\\\\n for i in $(seq 1 120); do kubectl get --raw=/readyz \\u003e/dev/null 2\\u003e\\u00261 \\u0026\\u0026 break; sleep 2; done ; \\\\\\n echo \\\"[pin-service-ips] ensure namespaces exist...\\\" ; \\\\\\n kubectl get ns kube-system \\u003e/dev/null ; \\\\\\n kubectl get ns argocd \\u003e/dev/null 2\\u003e\\u00261 || kubectl create ns argocd ; \\\\\\n kubectl get ns calico-system \\u003e/dev/null ; \\\\\\n echo \\\"[pin-service-ips] wait for coredns/argocd readiness (best effort)...\\\" ; \\\\\\n kubectl -n kube-system wait deploy coredns --for=condition=Available=True --timeout=300s || true ; \\\\\\n kubectl -n argocd wait deploy argocd-server --for=condition=Available=True --timeout=600s || true ; \\\\\\n echo \\\"[pin-service-ips] replace Services with fixed ClusterIPs...\\\" ; \\\\\\n kubectl -n kube-system delete svc kube-dns --ignore-not-found ; \\\\\\n kubectl apply -f /etc/kubernetes/addons/kube-dns-fixed-svc.yaml ; \\\\\\n kubectl -n argocd delete svc argocd-server --ignore-not-found ; \\\\\\n kubectl apply -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/argocd/service.yaml ; \\\\\\n kubectl -n calico-system delete svc whisker --ignore-not-found || true ; \\\\\\n kubectl create -f http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/calico-config/whisker.yaml || true ; \\\\\\n echo \\\"[pin-service-ips] done.\\\" \\\\\\n'\\n\\nExecStartPost=/usr/bin/touch /var/lib/undercloud-stamps/pin-service-ips.done\\nRestart=on-failure\\nRestartSec=120s\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"pin-service-ips.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "control_plane2_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::92/64\n Address=2001:470:7116:2::92/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.92/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8s.undercloud.local/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"fd00:0:0:2::92\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::92\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubeadm.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target", + "id": "1638069321", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQWuEMBAF4Pv8jd56yGay4rKBHIQtpYeKtEfxEJIpSjUpSdT674u21ULJ6b2Pl6mfdTJtA6UeSFFq7wHqktLsw3sDhbWBYlRvlnO5PiHlVZzybBfBOcrswuUFMT/47lEnmvXy38/wS38/PcOtfD0alBL3E8gZZ4JdxUlk+/in/N5tARnCzQ+6c1GNzlIwvR8t673RPTxVU14YQx/ppVDOb7kK3aTN8vCZyMXOu7jCVwAAAP//baxNNg8BAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwWobQQyG7/sUOjoBj68lN5MGWmgTaJJDe5M1ylp4drRImoXt0xd7c+jBNYX4v8xISB/fMLcfT3cL/8j29eXL04+vvx4+w/b+/uH5GZ4ev/08M3iB8d85MV724uCzBw8gDiNagL4BgjM14wxS3ww9rFE043SOsS0FkEImCWEHNIZBq4Qe97FmKNr3nNMlj9eKLfZq8vu4Q8TuoAaDeHM+mnmYUJQZRtO97CTe2X8xBpzB2FsJkApZnGQsUtHmxYJ7XES1pvMeV/jbbn3ldIe2Y6sc7EBaw7QwjAUrw6Nm7rrvWLFnmATvTrMUBVbvl5uOsAgprJbz1Jq4sCmsYYd0aCOslsZNh9YrZdhHjH632Szl2tkmtrRUySdKh0+eWs1sVLTlVJSwbK7/8D8BAAD//2cEMuhrAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSv27bPBTFdz4FofmTRFmy5XDzl2RoAxRGXaRD0YEir2xCMq/AP0qcpy8oqakddCu8GOI55L3nd8Sgn8E6jYbTLjQg1Dnrti7TmI9FA16UpNNGcfoZtblH0+pjsMJrNESi8Rb7fS8McEJpj1L0u/2nR6MG1MbHb5QKNYL12sFOKQvOcZq0ijEefyvO71bJJGu0UXu0ntNNVZWEUhldrZbCwxNcogugliWsZSuFrKGq60puoJWCVZtKVCu1vavbkrV1yVhZMbVZ19tmLdumEqUqqyYhBhV8haN2ft4gDmjEGThdVkmHuMsqvm71AWUHntNg9CvP89wGk0ed0AasuvqbOZQdoVN8PfjHV2/Fzh7dvH98NNUDp8mKsYJXNeN1UWyud5d9cB5sqkxMp2AZy+6KbF3+d+toecH5upw9I/bhDOnQh6M2qdKW0yTHwee9buAVZB6nsQY8uHwZbBG7fPbmkywhSjuJI9hLnLdB9DGe4Rt2YBaCgz6AHcG+g6XJj2uGBWM/ecQ2j+YnK0268VicZOZLCy842sFVq43FdhYF40QLh04P97tnsDPpqYXeBiBpmpKP1ezBZ3Jq4G1Di6WhT7PmtqTivXWcJ+QEovent/+1UbubgwXBA56FnmbfuiwYBVb2GFQ2VfuP6sshJpN+RHoFKL3mmBB5tBiGB6tHiKDcxXk4q4SSGcV+AvPwDxBF8Ccw/neIhFJh0FzOGJYaghFND2qJl9IXaE6I3d8O411o9dv7VWdUwOn32UF+BQAA//8OJfhXMgQAAA==\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubeadm.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "control_plane3_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n control-plane3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::93/64\n Address=2001:470:7116:2::93/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.93/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes controle plane Node\n\n Manage via:\n kubectl (kubectl)\n calico (calicoctl)\n velero - backup (velero)\n argocd https://argocd-server.argocd.svc.k8aux.undercloud.cf/\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n controlPlane:\n localAPIEndpoint:\n advertiseAddress: \"2001:470:7116:2::93\"\n bindPort: 6443\n certificateKey: \"fee7c3e5cfcac7e4774c6efca0464a42d897f30f7300340d6578b5cfb4a3d34b\"\n nodeRegistration:\n name: control-plane3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::93\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n authentication:\n anonymous:\n enabled: true\n webhook:\n enabled: true\n authorization:\n mode: Webhook\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=multi-user.target\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n\n # copy files for kubectl\n ExecStartPost=/usr/bin/mkdir -p /home/core/.kube\n ExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\n ExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target", + "id": "1600146304", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,control-plane3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2TNQUvEMBAF4Pv8DW8espmkdNlADoUV8WBZ9LjsISQjLbaJJGlr/720aitITu99vMz12WTb3KA2PWnKzT3AtaY8hfh+g8q5SCnpN8e5Wp5Q6iQPZbGJ4BxVceTqiFjufPdoMk1m/u8SfunvpxLO9eveoFK4nUDOOBPsJA+i2MY/5fduDcgQzqE3rU968I6i7cLgWBes6eDpMpaVtfSRXyrtw5ovsR2NnR8+M/nUBp8W+AoAAP//dV+cVw8BAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwYobMQyG7/MUOmYX4lzL3sJ2oYV2A01yaG+KrMyIeOxBkodOn74kk0MPaSg0/8WWkD4+4+f/T/MMf8l6v/u0+fb5x9tHWL++vm23sHn/8v3G4B3GP+fC2HViYJM59yAGA6pDOQKCMVXlCJKPiuZayatyuMVYpwRILqO4sAEqQ1+yeDnvY46QSttyDPc89hmrd0Xl13mHiM2gKPRi1fhsZq5CniYYtHRyEL+y/2D0OIGy1eQgGaIYyZAko06zBbc4i5Ycbns84G+b5YPTnOqBNbOzAZXsWhLDkDAzvJfITfMVM7YMo+DLZZY8weJ6eWoIk1CBxXxeWiMn1gJLOCCd6gCLufHUoLaFInTug72sVnO5NNaRNcxVsJHC6QPWn6HmyEqp1BjouHr8u38HAAD//04R3qpqAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SSP2/bMBDFd34KQnMlUZZsOdzcJEMboDDqIh2KDhR5sgnJPIF/lDifvqCkpnFQdCm8GOI78t37PTHoR7BOo+G0Cw0Idc66rcs05mPRgBcl6bRRnH5GbW7RtPoYrPAaDZFovMV+3wsDnFDaoxT9bv/p3qgBtfHxG6VCjWC9drBTyoJznCYrxgpe1YzXRbHhK85vymTSNtqoPVrP6aaqSkKpjKOtlsLDA1w4TVqAWpawlq0Usoaqriu5gVYKVm0qUa3U9qZuS9bWJWNlxdRmXW+btWybSpSqrJqEGFTwFY7a+XmN6NKIM3C67JMOcaHpdasPKDvwnAajn3me5zaYPOqENmDVm7+ZQ9kROmXYg79/9lbs7NHNIcRHUz38a3fZB+fBpsrEiAqWseymyNblh+uJlhecr5eZEftwhnTow1GbVGnLaZLj4PNeN/AMMo9urAEPLl+MLWKXz7P5JEuI0k7iCPYS/TaIPsYzfMMOzIJx0AewI9hXujT50SrGePytOC8Y+8kjttman0Zp0o3H4iQzX1p4wtEOrlptLLazKBgnWjh0erjdPYKdSU9V9DYASdOUvO9nDz6TUw2va1osNX2YNddNFa/V4zwhJxC9P7181Ebtrg4WBHd4FnryvnVZMAqs7DGobOr3H9WXQ0wmfY/0DaD0LceEyKPFMNxZPUIE5S7Ow1kllMwo9hOYu/+AKII/gfG/QySUCoPmcsaw1BCMaHpQS7yUPkFzQuz+dhjvQqtfXq86owJOv88T5FcAAAD//xliG4M3BAAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n# copy files for kubectl\\nExecStartPost=/usr/bin/mkdir -p /home/core/.kube\\nExecStartPost=/usr/bin/cp -i /etc/kubernetes/admin.conf /home/core/.kube/config\\nExecStartPost=/usr/bin/chown core:core /home/core/.kube/config\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "worker1_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker1\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::101/64\n Address=2001:470:7116:2::101/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.101/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker1\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::101\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "393379439", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker1%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nShUAOhRXxYFn0uOwhJCMttokkaWv/vbRqLSxzeu+b997lxWTbXKE2PWvOzT3ApeY8hfhxhcq5yCnpd4eolpNKEdJDWWxIIpIqjqiOROWO3z2ZzJOZbx8O8If2tQc41W//DilF2wahQCHF0iyLLf3r/gRXQYLgFHrT+qQH7zjaLgxOdMGaDp7PY1lZy5/5tdI+rPoc29HY+fErs09t8GkB3wEAAP//4pHcSRIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRz27bPBDE73oKQudPf2g7Tr69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+dnBnpFCcQeVJeOaNy57J5CSVyN+ojRrLOOvAP1mck/s2+oTWIisc88O/yOLYW4HECmlDdnBPXG0qHoTCkrdGDbYQSVPF2gqipJvrLsoyGP4m62ZWDbZWqO0WN8uUQxO2nD9K5S028FDaDyVV1r2DzW8Kj1FlYAutb5rLF9ChGlcD6AynVd1uX/unxY/3dvaUADPKwXz8h9OmMx9KklXzgSUHnFQ6x6OuIFbTXFEY8RQ3VNdhWHavFWsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV9cwrXmQ+hdsN5trtDhbVd6NrT7ZMq4F33iUIWxWW+FmESUfTIOHjobn3SsKNWQXGipKwqwoiuwj4x5jaWeU96j1FfWXRXNP2zgnGKZeAfLshKaPp9+fyLvd3cUVwZ7PhubsT6FM3qHYnpMre7am/6v6epiaKT4yvQFU3HLMM9sKp2EvNOIEKryHiGeXq2xB8W0Gs/8HiH8CAAD//1pu0RwQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "worker2_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker2\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::102/64\n Address=2001:470:7116:2::102/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.102/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker2\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::102\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "3758288308", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker2%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nShUAOhRXxYFn0uOwhJCMttokkaWv/vbRqLSxzeu+b997lxWTbXKE2PWvOzT3ApeY8hfhxhcq5yCnpd4eolpNKEcqHstiQRCRVHFEdicodv3symScz3z4c4A/taw9wqt/+HVKKtg1CgUKKpVkWW/rX/QmuggTBKfSm9UkP3nG0XRic6II1HTyfx7Kylj/za6V9WPU5tqOx8+NXZp/a4NMCvgMAAP//yrsJeBIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRwW7bPBCE73oKQudfomQ7Tv69uXEuLVAUdZFL0QNNruSFZK6wJBWnT19IMlA714IXgpwhZ+czA72iBGIPqktHNO5cdk+hJNZjfcRo1llH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VBWmVJW6MC2wwgqebqA1lqS15Z9NORR3M22DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVS+qqoaNo8VPNb1FlYAdbXKZ43tU4gohfMBVF5XZVX+X5cP6//uLQ3UAA/rxTNyn85YDH1qyReOBFSueYi6pyNe0OopjniMGPQ12VUc9OLVsyzPHAXLI8r7FPjIHKdihh/coV9GMAMdUEaUF+8GJh9B5T8bV1UwrXmQ6hdsN5trtDhbVd6NbX2yZVwLvvEoQ9istsLNIko+mAYPHQ3Pu1cUasguNFSUhFlRFNlHxj3G0s4o71HXV9RfFs09beOcYJh6BcizE5o+nn5/Iu92dxdXBHs+G5qzP4UyeYdie06u7Nma/q/q62FqpvjI9AZQccsxz2wrnIa90IgTqPAeIp5drrIFxbcZzP4fIP4JAAD//3XfDwQQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "worker3_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker3\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::103/64\n Address=2001:470:7116:2::103/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.103/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker3\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::103\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "4172286456", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker3%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM2npQiCHwop4sCx6XPYQkpEttokkaWv/vbRqLSxzeu+b9975xWR7vUBjetacr/cA54bzFOLHBWrnIqek3x2iWk4qRVg8VOWGJCKp8oDqQFTt+N2TyTyZ+fahgD+0ry3g2Lz9O6QUbRuEAoUUS7Mst/Sv+xNcBQmCY+hN65MevONouzA40QVrOng+jVVtLX/m11r7sOpTbEdj58evzD61wacFfAcAAP//7V9q3hIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRPY/bPBCEe/0KQvUrUTr7Pt7tnPM1CRAEcXBNkIImV/JCMldYkjpffn2gDyD2tQEbgpwhZ+cxA72iBGIPqktHNO5cdk+hJNZjfcRoNllH3oH6zOSf2TfUJjGR2GeeHX7HlkJcDiBTypszgnpj6VA2mVJW6MC2wwgqebqA1lqS15Z9NORR3NW2DGy7TM0xeowvlyhmJ22Y3lVq+q2gAVR+V1U1bB8reKzrB7gDqKtNPmtsn0JEKZwPoPK6Kqvy/7q83/x3a2mgBrhfPSP36YzF0KeWfOFIQOWah6h7OuIFrZ7iiMeIQa/JVnHQi1fPsjxzFCyPKO9T4CNznIoZfnCHfhnBDHRAGVFevBuYfASV/2xcVcG05kGqX/Cw3a7R4mxVeTe29cmWcSP4xqMMYXv3INwsouSDafDQ0fC8e0WhhuxCQ0VJmBVFkX1k3GMs7YzyFnW9ov6yaG5pG+cEw9QrQJ6d0PTx9PsTebe7uVgR7PlsaM7+FMrkHYrtObmyZ2v6v6qvh6mZ4iPTK0DFNcc8s61wGvZCI06gwnuIeHa5yhYU32Yw+3+A+CcAAP//r02VuhADAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "data", + "type": "ct_config", + "name": "worker4_ignition", + "provider": "provider[\"registry.terraform.io/poseidon/ct\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "content": "variant: flatcar\nversion: 1.1.0\n\npasswd:\n users:\n - name: core\n ssh_authorized_keys:\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\"\n - \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n\nstorage:\n directories:\n - path: /opt/bin\n overwrite: true\n mode: 0755\n - path: /opt/cni/bin\n overwrite: true\n mode: 0755\n files:\n - path: /etc/hostname\n mode: 0644\n contents:\n inline: |\n worker4\n\n - path: /etc/systemd/network/00-eth.network\n mode: 0644\n contents:\n inline: |\n [Match]\n Name=eth*\n \n [Network]\n Address=fd00:0:0:2::104/64\n Address=2001:470:7116:2::104/64\n #Gateway=2001:470:7116:2::3\n Gateway=fd00:0:0:2::3\n DNS=fd00:0:0:1::1\n Address=10.0.2.104/24\n Gateway=10.0.2.3\n DNS=10.0.1.1\n Domains=undercloud.local\n IPv6AcceptRA=no\n IPv6PrivacyExtensions=no\n\n - path: /etc/hosts\n mode: 0644\n overwrite: true\n contents:\n inline: |\n 127.0.0.1 localhost\n ::1 localhost ip6-localhost ip6-loopback\n\n 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1\n 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2\n 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3\n 2001:470:7116:2::101 worker1.undercloud.local worker1\n 2001:470:7116:2::102 worker2.undercloud.local worker2\n 2001:470:7116:2::103 worker3.undercloud.local worker3\n 2001:470:7116:2::104 worker4.undercloud.local worker4\n\n fd00:0:0:2::91 control-plane1.undercloud.local control-plane1\n fd00:0:0:2::92 control-plane2.undercloud.local control-plane2\n fd00:0:0:2::93 control-plane3.undercloud.local control-plane3\n fd00:0:0:2::101 worker1.undercloud.local worker1\n fd00:0:0:2::102 worker2.undercloud.local worker2\n fd00:0:0:2::103 worker3.undercloud.local worker3\n fd00:0:0:2::104 worker4.undercloud.local worker4\n\n 10.0.2.91 control-plane1.undercloud.local control-plane1\n 10.0.2.92 control-plane2.undercloud.local control-plane2\n 10.0.2.93 control-plane3.undercloud.local control-plane3\n 10.0.2.101 worker1.undercloud.local worker1\n 10.0.2.102 worker2.undercloud.local worker2\n 10.0.2.103 worker3.undercloud.local worker3\n 10.0.2.104 worker4.undercloud.local worker4\n\n - path: /etc/motd\n mode: 0644\n overwrite: true\n contents:\n inline: |\n *******************************************************************\n * AUTHORIZED ACCESS ONLY *\n * *\n * This system is part of a secured infrastructure. *\n * All activities are monitored and logged. *\n * Unauthorized access or misuse is strictly prohibited and *\n * may result in disciplinary and legal action. *\n *******************************************************************\n\n --------------------------------------------------------------------------------\n kubernetes worker Node\n\n dont manage\n --------------------------------------------------------------------------------\n\n - path: /etc/sysctl.d/99-k8s.conf\n mode: 0644\n contents:\n inline: |\n net.ipv4.ip_forward = 1\n net.ipv6.ip_forward = 1\n net.ipv6.conf.all.forwarding = 1\n net.ipv4.conf.all.forwarding = 1\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.netfilter.nf_conntrack_max = 1000000\n net.ipv4.conf.all.rp_filter = 0\n net.ipv6.conf.all.disable_ipv6 = 0\n vm.overcommit_memory = 1\n fs.inotify.max_user_watches = 524288\n fs.inotify.max_user_instances = 4096\n kernel.panic = 10\n kernel.panic_on_oops = 1\n net.ipv6.conf.all.accept_ra = 2\n\n - path: /etc/flatcar/update.conf\n overwrite: true\n mode: 0420\n contents:\n inline: |\n REBOOT_STRATEGY=off\n\n - path: /opt/bin/kubeadm\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\"\n\n - path: /opt/bin/kubelet\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\"\n\n - path: /opt/bin/kubectl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\"\n\n - path: /opt/bin/calicoctl\n mode: 0755\n contents:\n source: \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\"\n\n - path: /etc/kubernetes/kubeadm-join.yaml\n mode: 0644\n contents:\n inline: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: JoinConfiguration\n nodeRegistration:\n name: worker4\n criSocket: unix:///run/containerd/containerd.sock\n kubeletExtraArgs:\n node-ip: \"2001:470:7116:2::104\"\n cluster-dns: \"10.0.91.53,2001:470:7116:f:1::53\"\n volume-plugin-dir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n discovery:\n bootstrapToken:\n apiServerEndpoint: \"[fd00:0:0:2::100]:6443\"\n token: \"kvg1hc.t3rewovrps426rof\"\n unsafeSkipCAVerification: true\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n address: \"::\"\n healthzBindAddress: \"::\"\n clusterDomain: \"k8s.undercloud.local\"\n clusterDNS:\n - \"2001:470:7116:f:1::53\"\n - \"10.0.91.53\"\n cgroupDriver: \"systemd\" \n volumePluginDir: \"/opt/libexec/kubernetes/kubelet-plugins/volume/exec/\"\n\nsystemd:\n units:\n - name: modules-load.service\n enabled: true\n contents: |\n [Unit]\n Description=Load necessary kernel modules\n Before=containerd.service kubeadm-init.service\n\n [Service]\n Type=oneshot\n ExecStart=/usr/bin/modprobe br_netfilter\n ExecStart=/usr/bin/modprobe overlay\n RemainAfterExit=yes\n\n [Install]\n WantedBy=multi-user.target\n\n - name: systemd-networkd-wait-online.service\n enabled: true\n\n - name: containerd.service\n enabled: true\n contents: |\n [Unit]\n Description=containerd container runtime\n After=network.target modules-load.service\n\n [Service]\n ExecStart=/usr/bin/containerd\n Restart=always\n RestartSec=5\n Delegate=yes\n KillMode=process\n OOMScoreAdjust=-999\n\n [Install]\n WantedBy=multi-user.target\n\n - name: set-timezone.service\n enabled: true\n contents: |\n [Unit]\n Description=Set Timezone\n After=network-online.target\n Wants=network-online.target\n [Service]\n StandardOutput=journal+console\n StandardError=journal+console\n Type=oneshot\n Restart=on-failure\n ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\n ExecStart=/usr/bin/timedatectl set-ntp true \n [Install]\n WantedBy=kubelet.service\n\n - name: kubelet.service\n enabled: true\n contents: |\n [Unit]\n Description=kubelet, the Kubernetes Node Agent\n Documentation=https://kubernets.io/docs/home\n Wants=network-online.target\n After=network-online.target\n [Service]\n #StandardOutput=journal+console\n #StandardError=journal+console\n #EnvironmentFile=/run/metadata/coreos\n Environment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\n Environment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\n # This is a file that \"kubeadm init\" and \"kubeadm join\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\n EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\n ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n Restart=always\n StartLimitInterval=0\n RestartSec=10\n [Install]\n WantedBy=multi-user.target\n\n - name: kubeadm-join.service\n enabled: true\n contents: |\n [Unit]\n Description=Join node to Kubernetes cluster\n After=network-online.target containerd.service kubelet.service\n Wants=network-online.target\n\n [Service]\n Type=oneshot\n # Environment\n Environment=KUBECONFIG=/etc/kubernetes/admin.conf\n Environment=DATASTORE_TYPE=kubernetes\n Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\n \n ExecStartPre=/bin/sleep 30s\n\n ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\n \n #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\n Restart=on-failure\n RestartSec=120s\n\n [Install]\n WantedBy=multi-user.target\n\n - name: ping-all.service\n enabled: true\n contents: |\n [Unit]\n Description=Ping cluster nodes every 5 seconds\n After=network-online.target\n Wants=network-online.target\n\n [Service]\n Type=simple\n ExecStart=/usr/bin/bash -c '\\\n while true; do \\\n ping -c1 -W1 fd00:0:0:2::91; \\\n ping -c1 -W1 fd00:0:0:2::92; \\\n ping -c1 -W1 fd00:0:0:2::93; \\\n ping -c1 -W1 fd00:0:0:2::101; \\\n ping -c1 -W1 fd00:0:0:2::102; \\\n ping -c1 -W1 fd00:0:0:2::103; \\\n ping -c1 -W1 fd00:0:0:2::104; \\\n ping -c1 -W1 2001:470:7116:2::91; \\\n ping -c1 -W1 2001:470:7116:2::92; \\\n ping -c1 -W1 2001:470:7116:2::93; \\\n ping -c1 -W1 2001:470:7116:2::101; \\\n ping -c1 -W1 2001:470:7116:2::102; \\\n ping -c1 -W1 2001:470:7116:2::103; \\\n ping -c1 -W1 2001:470:7116:2::104; \\\n sleep 5; \\\n done'\n Restart=always\n RestartSec=5\n\n [Install]\n WantedBy=multi-user.target\n", + "id": "1542817396", + "pretty_print": true, + "rendered": "{\n \"ignition\": {\n \"config\": {\n \"replace\": {\n \"verification\": {}\n }\n },\n \"proxy\": {},\n \"security\": {\n \"tls\": {}\n },\n \"timeouts\": {},\n \"version\": \"3.4.0\"\n },\n \"kernelArguments\": {},\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"core\",\n \"sshAuthorizedKeys\": [\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud\",\n \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmweMOyvxl6Z5UzMKLJI1va+w8IlTl0tAGQkjGebiHb sebastian@undercloud-flatcar\"\n ]\n }\n ]\n },\n \"storage\": {\n \"directories\": [\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/bin\",\n \"user\": {},\n \"mode\": 493\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/opt/cni/bin\",\n \"user\": {},\n \"mode\": 493\n }\n ],\n \"files\": [\n {\n \"group\": {},\n \"path\": \"/etc/hostname\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,worker4%0A\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/systemd/network/00-eth.network\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/2SNQUvEMBCF7/M3vHmIM9nQhUAOhRXxYFn0uOwhJCNb7CaSpK3999KqtbDM6b1v3nunF1vc5QyNvbLhcrkHODVcxpg+zlB7nzhn8+4R9XxSa0L1UKkVSUTSao96T1Rt+N2TLTza6fZhB39oW7uDQ/P275DWtG4QChRSzM1Srelf9ye4CBIEh3i1bcimD56T62LvRRed7eD5OFS1c/xZXmsT4qKPqR2smx6/CofcxpBn8B0AAP//mu+jGxIBAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/hosts\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/5STUW6DMAxA/3OKXKBRbEdF820YMG1qRBBrtetPYQERZgikX9R5T62fAKyMNdaA1tqHpvaf4fupmOPz31m+1V/D/bZ9CsN73TyUQmuBXWW5ArgzMr+B1k3on2Pwt8HXfQfm1bfd2Pjwas2k2cwFB24cWHCg4KCNgwoO+u8AC/onjI9uFP5EGkgUpiHuUShRlIa0R4m/0KWh26OcUh+ttRw/qU88FxtlChQU5USZggRFudBaEePEUwyUQ7iCDvrkEK2ggzw55FbQUR2I7yGaKczFKjOK12vMKF2vkNApQHH7y2U8sfXlMp3Y9nLZndjybwAAAP//DKgrvfAEAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/motd\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6yRwUoDMRCG7/sU/7lg36HUgoK0YNuD3qbJdDs0m5SZibI+vaz14GEtgvud83/5YGb/p5nhFxb73cPm+fF1dY/FcrnabrFZP72MPLzh+DNfjt1JDNabcwcxXEgd5QiCcajKEZKPSuZag1fl+ZhjkRIouLyJCxtIGV3J4mXYU45IpW05zm917DNVPxWVj2ETApuhKDqxajyUmasETz0uWk5yEP92/3B01EPZanJIRhQLckmSSftrBbd0DS15Pt4xwW2bu4lpzvXAmtnZ8F70zIp1idw0sWRHR5lanv7TzwAAAP//EeTc++cCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"path\": \"/etc/sysctl.d/99-k8s.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/4zRUW7DIAwG4PecIhcIyqIs6h56FsslprMKBhmWtrefsmVZN2Va/hck9CHzg1AxnKbecAIX9Yo61r9zrJ+qxQ27nY3iDHpvFs1y3nD9HndSHs+0LI24xqL3DaeCJ0/5fzcs8MsJFce+kBpxYKNIUbQXCHhb57Yf2bilJvg8+9C63Wg9cp6Hwrz77aZg4kRqYwhcIFCIet96RZcNSyzs7ibgDd4yKVyx2Ne57uqeu747HDYxSy4oduXHum9fhupCKuRNQmFb/5G5/Q8IUSDGlPd9N1pLqYDig+uq9wAAAP//zXOeZmcCAAA=\",\n \"verification\": {}\n },\n \"mode\": 420\n },\n {\n \"group\": {},\n \"overwrite\": true,\n \"path\": \"/etc/flatcar/update.conf\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"\",\n \"source\": \"data:,REBOOT_STRATEGY%3Doff%0A\",\n \"verification\": {}\n },\n \"mode\": 272\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubeadm\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubelet\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/kubectl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/opt/bin/calicoctl\",\n \"user\": {},\n \"contents\": {\n \"source\": \"http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl\",\n \"verification\": {}\n },\n \"mode\": 493\n },\n {\n \"group\": {},\n \"path\": \"/etc/kubernetes/kubeadm-join.yaml\",\n \"user\": {},\n \"contents\": {\n \"compression\": \"gzip\",\n \"source\": \"data:;base64,H4sIAAAAAAAC/6SRv47bPBDEez0FofqTKJ11f77tnPM1CRAEcXBNkIImV/JCMldYkjpfnj6QZCD2tQEbgpwhZ+dnRnpFCcQeVJ8OaNyp7J9CSayn+oDRbLKevAP1mck/s2+pS2Iisc88O/yOHYW4HkCmlDcnBPXG0qM0mVJWaM+2xwgqeTqD1lqS15Z9NORR3NW2DGz7TC0xBowv5yhmK12Y31Vq/q2gEVR+V1U1NI8VPNb1A9wB1FWTLxo7pBBRCucDqLyuyqr8vy7vN//dWlqoAe43q2fiIZ2wGIfUkS8cCahc8xj1QAc8o9VzHPEYMehLsos46NWrF1meOQqWJ5T3OfCBOc7FjD+4R7+OYEbao0woL96NTD6Cyn+2rqpgXssg1S94aJpLtLhYVd5PXX20ZdwIvvEkY2juHoTbVZR8MC3uexqft68o1JJdaagoCbOiKLKPjAeMpV1Q3qKuL6i/rJpb2sY5wTD3CpBnRzRDPP7+RN5tby4uCHZ8MrRkfwpl8g7FDpxcObA1w1/V1/3cTPGR6RWg4ppjntlOOI07oQlnUOE9RDy5XGUrim8LmN0/QPwTAAD//yu9sjUQAwAA\",\n \"verification\": {}\n },\n \"mode\": 420\n }\n ]\n },\n \"systemd\": {\n \"units\": [\n {\n \"contents\": \"[Unit]\\nDescription=Load necessary kernel modules\\nBefore=containerd.service kubeadm-init.service\\n\\n[Service]\\nType=oneshot\\nExecStart=/usr/bin/modprobe br_netfilter\\nExecStart=/usr/bin/modprobe overlay\\nRemainAfterExit=yes\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"modules-load.service\"\n },\n {\n \"enabled\": true,\n \"name\": \"systemd-networkd-wait-online.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=containerd container runtime\\nAfter=network.target modules-load.service\\n\\n[Service]\\nExecStart=/usr/bin/containerd\\nRestart=always\\nRestartSec=5\\nDelegate=yes\\nKillMode=process\\nOOMScoreAdjust=-999\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"containerd.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Set Timezone\\nAfter=network-online.target\\nWants=network-online.target\\n[Service]\\nStandardOutput=journal+console\\nStandardError=journal+console\\nType=oneshot\\nRestart=on-failure\\nExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin\\nExecStart=/usr/bin/timedatectl set-ntp true \\n[Install]\\nWantedBy=kubelet.service\\n\",\n \"enabled\": true,\n \"name\": \"set-timezone.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=kubelet, the Kubernetes Node Agent\\nDocumentation=https://kubernets.io/docs/home\\nWants=network-online.target\\nAfter=network-online.target\\n[Service]\\n#StandardOutput=journal+console\\n#StandardError=journal+console\\n#EnvironmentFile=/run/metadata/coreos\\nEnvironment=\\\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\\\"\\nEnvironment=\\\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\\\"\\n# This is a file that \\\"kubeadm init\\\" and \\\"kubeadm join\\\" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically\\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\\nRestart=always\\nStartLimitInterval=0\\nRestartSec=10\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubelet.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Join node to Kubernetes cluster\\nAfter=network-online.target containerd.service kubelet.service\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\n# Environment\\nEnvironment=KUBECONFIG=/etc/kubernetes/admin.conf\\nEnvironment=DATASTORE_TYPE=kubernetes\\nEnvironment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/\\n\\nExecStartPre=/bin/sleep 30s\\n\\nExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml\\n\\n#ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service\\nRestart=on-failure\\nRestartSec=120s\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"kubeadm-join.service\"\n },\n {\n \"contents\": \"[Unit]\\nDescription=Ping cluster nodes every 5 seconds\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/bash -c '\\\\\\nwhile true; do \\\\\\nping -c1 -W1 fd00:0:0:2::91; \\\\\\nping -c1 -W1 fd00:0:0:2::92; \\\\\\nping -c1 -W1 fd00:0:0:2::93; \\\\\\nping -c1 -W1 fd00:0:0:2::101; \\\\\\nping -c1 -W1 fd00:0:0:2::102; \\\\\\nping -c1 -W1 fd00:0:0:2::103; \\\\\\nping -c1 -W1 fd00:0:0:2::104; \\\\\\nping -c1 -W1 2001:470:7116:2::91; \\\\\\nping -c1 -W1 2001:470:7116:2::92; \\\\\\nping -c1 -W1 2001:470:7116:2::93; \\\\\\nping -c1 -W1 2001:470:7116:2::101; \\\\\\nping -c1 -W1 2001:470:7116:2::102; \\\\\\nping -c1 -W1 2001:470:7116:2::103; \\\\\\nping -c1 -W1 2001:470:7116:2::104; \\\\\\nsleep 5; \\\\\\ndone'\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\",\n \"enabled\": true,\n \"name\": \"ping-all.service\"\n }\n ]\n }\n}", + "snippets": null, + "strict": false + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "wait_for_cluster_ready", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2188697122693509275", + "triggers": null + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "data.ct_config.control_plane3_ignition", + "null_resource.wait_for_cp1_api", + "null_resource.wait_for_cp2_cp3_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.control_plane2", + "proxmox_virtual_environment_vm.control_plane3", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "wait_for_cp1_api", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "7332826353539929608", + "triggers": null + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "wait_for_cp2_cp3_api", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "8339091622691904298", + "triggers": null + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "data.ct_config.control_plane3_ignition", + "null_resource.wait_for_cp1_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.control_plane2", + "proxmox_virtual_environment_vm.control_plane3", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_download_file", + "name": "flatcar_image", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "checksum": null, + "checksum_algorithm": null, + "content_type": "import", + "datastore_id": "cephfs", + "decompression_algorithm": null, + "file_name": "flatcar_production_proxmoxve_image.qcow2", + "id": "cephfs:import/flatcar_production_proxmoxve_image.qcow2", + "node_name": "hyper1", + "overwrite": true, + "overwrite_unmanaged": false, + "size": 573243392, + "upload_timeout": 600, + "url": "http://git.undercloud.local:3000/admin/undercloud-infrastructure/raw/branch/main/iso/flatcar_production_proxmoxve_image.img", + "verify": true + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "eyJvcmlnaW5hbF9zdGF0ZV9zaXplIjoiTlRjek1qUXpNemt5In0=" + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "control_plane1", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 4, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes control-plane1", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1001", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/control-plane1-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.91" + ], + [], + [], + [ + "10.0.10.0" + ], + [], + [], + [], + [], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::91", + "fd00:0:0:2::91", + "fe80::be24:11ff:feb2:a725" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:8495:c8a0:d59b:d0c0" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:B2:A7:25", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:78:AC:F8:81:98", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" + ], + "machine": null, + "memory": [ + { + "dedicated": 8192, + "floating": 8192, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "control-plane1", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:B2:A7:25", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "calieae95b74f2c", + "cali7c65858f1d3", + "tunl0", + "vxlan-v6.calico", + "caliddba80e9b5e", + "cali58a55eebea1", + "calife47df6812f", + "calicb53b7a002e" + ], + "node_name": "hyper1", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "control-plane", + "flatcar", + "kubernetes", + "terraform" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1001, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "control_plane2", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 4, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes control-plane2", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1002", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/control-plane2-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.92" + ], + [], + [ + "10.0.10.64" + ], + [], + [], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::92", + "fd00:0:0:2::92", + "fe80::be24:11ff:fee4:d5d6" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:63eb:d7a8:d9b3:1e00" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:E4:D5:D6", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:42:B5:2A:6A:1B", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" + ], + "machine": null, + "memory": [ + { + "dedicated": 8192, + "floating": 8192, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "control-plane2", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:E4:D5:D6", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "calia2e2dfb30a3", + "tunl0", + "vxlan-v6.calico", + "cali25103b133f8", + "calib7a33493c64" + ], + "node_name": "hyper2", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "control-plane", + "flatcar", + "kubernetes", + "terraform" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1002, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "null_resource.wait_for_cp1_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "control_plane3", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 4, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes control-plane3", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1003", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/control-plane3-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.93" + ], + [], + [], + [ + "10.0.10.192" + ], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::93", + "fd00:0:0:2::93", + "fe80::be24:11ff:fe69:4638" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:edec:9c79:57a2:840" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:69:46:38", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:EE:86:C9:B8:69" + ], + "machine": null, + "memory": [ + { + "dedicated": 8192, + "floating": 8192, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "control-plane3", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:69:46:38", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "cali6ae1ad1f505", + "calicfb3e89fc4e", + "tunl0", + "vxlan-v6.calico" + ], + "node_name": "hyper3", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "control-plane", + "flatcar", + "kubernetes", + "terraform" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1003, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane3_ignition", + "null_resource.wait_for_cp1_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "flatcar_template", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [ + { + "enabled": true, + "timeout": "15m", + "trim": false, + "type": "virtio" + } + ], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0", + "ide2" + ], + "cdrom": [], + "clone": [], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 1, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "managed by terraform - base template for flatcar", + "disk": [ + { + "aio": "io_uring", + "backup": true, + "cache": "writeback", + "datastore_id": "Pool1", + "discard": "on", + "file_format": "raw", + "file_id": "", + "import_from": "cephfs:import/flatcar_production_proxmoxve_image.qcow2", + "interface": "virtio0", + "iothread": false, + "path_in_datastore": "base-999-disk-0", + "replicate": true, + "serial": "", + "size": 50, + "speed": [], + "ssd": false + } + ], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "999", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [], + "ipv6_addresses": [], + "keyboard_layout": "en-us", + "kvm_arguments": "", + "mac_addresses": [ + "BC:24:11:8D:8B:89" + ], + "machine": "", + "memory": [ + { + "dedicated": 2048, + "floating": 2048, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "flatcar-template", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:8D:8B:89", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [], + "node_name": "hyper1", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": null, + "startup": [], + "stop_on_destroy": true, + "tablet_device": true, + "tags": [ + "flatcar", + "kubernetes", + "terraform" + ], + "template": true, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 999, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "proxmox_virtual_environment_download_file.flatcar_image" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "worker1", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 14, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes worker1", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1011", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/worker1-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.101" + ], + [ + "10.0.10.103" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::101", + "fd00:0:0:2::101", + "fe80::be24:11ff:fe97:6379" + ], + [], + [ + "2001:470:7116:a:4e89:604c:ea1a:a982" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:97:63:79", + "00:00:00:00:00:00", + "66:ED:A6:2D:2B:8D", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" + ], + "machine": null, + "memory": [ + { + "dedicated": 25000, + "floating": 25000, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "worker1", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:97:63:79", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "tunl0", + "vxlan-v6.calico", + "cali438de1772bb", + "cali150b6f155d8", + "cali61a9554ce92", + "cali97e829f5c20", + "cali71eae69175a", + "cali482326879bd", + "cali471d98baa74", + "cali19f1a84797b", + "cali4c30009dc88", + "cali2fc5f4481bb" + ], + "node_name": "hyper4", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "flatcar", + "kubernetes", + "terraform", + "worker" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1011, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "data.ct_config.control_plane3_ignition", + "data.ct_config.worker1_ignition", + "null_resource.wait_for_cluster_ready", + "null_resource.wait_for_cp1_api", + "null_resource.wait_for_cp2_cp3_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_file.worker1_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.control_plane2", + "proxmox_virtual_environment_vm.control_plane3", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "worker2", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 6, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes worker2", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1012", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/worker2-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.102" + ], + [], + [ + "10.0.10.128" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::102", + "fd00:0:0:2::102", + "fe80::be24:11ff:fec2:408a" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [], + [ + "2001:470:7116:a:2903:60e7:598f:e9c0" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:C2:40:8A", + "EE:EE:EE:EE:EE:EE", + "00:00:00:00:00:00", + "66:65:83:E5:BB:CC", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" + ], + "machine": null, + "memory": [ + { + "dedicated": 14000, + "floating": 14000, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "worker2", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:C2:40:8A", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "calibce684fc6dd", + "tunl0", + "vxlan-v6.calico", + "calic5dffe1f987", + "cali9aa1fde2946", + "cali8512da79659", + "cali84cad734eaa", + "caliaa9e304886c", + "calidc5d61e997c", + "calidd602456e41", + "cali362ed739e39", + "cali17dc2dabdfa", + "cali8d2065ea893", + "calibb76b3ab372", + "cali933e1efdc43", + "cali2e499c0605e", + "cali46a83cb5b38", + "cali4b44079d8c4", + "cali8d3cee2bc69", + "cali2a4eb726f83", + "caliadd23907588", + "cali2ffe1c36b3b", + "cali0816a005e97", + "caliba9d140bc58", + "calic9eae6671be", + "califd0c1ceda99", + "cali410d4ce3d0a", + "cali119fc794523", + "cali815f9f28e57", + "cali561dcc29b32", + "cali3ef69860863", + "cali87a86cd44ac", + "cali22b64eb4620", + "calic15955a29e3", + "calie94c3337bf8", + "cali272f9b8fb79", + "calic43f18bdbfe", + "cali21e0604c323", + "cali48a28da992f", + "calia5cad583db5", + "calibef07565dae" + ], + "node_name": "hyper6", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "flatcar", + "kubernetes", + "terraform", + "worker" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1012, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "data.ct_config.control_plane3_ignition", + "data.ct_config.worker2_ignition", + "null_resource.wait_for_cluster_ready", + "null_resource.wait_for_cp1_api", + "null_resource.wait_for_cp2_cp3_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_file.worker2_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.control_plane2", + "proxmox_virtual_environment_vm.control_plane3", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + }, + { + "mode": "managed", + "type": "proxmox_virtual_environment_vm", + "name": "worker4", + "provider": "provider[\"registry.terraform.io/bpg/proxmox\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acpi": true, + "agent": [], + "amd_sev": [], + "audio_device": [], + "bios": "seabios", + "boot_order": [ + "virtio0" + ], + "cdrom": [], + "clone": [ + { + "datastore_id": "", + "full": true, + "node_name": "hyper1", + "retries": 1, + "vm_id": 999 + } + ], + "cpu": [ + { + "affinity": "", + "architecture": "", + "cores": 4, + "flags": [], + "hotplugged": 0, + "limit": 0, + "numa": false, + "sockets": 1, + "type": "host", + "units": 1024 + } + ], + "description": "kubernetes worker4", + "disk": [], + "efi_disk": [], + "hook_script_file_id": null, + "hostpci": [], + "id": "1014", + "initialization": [ + { + "datastore_id": "Pool1", + "dns": [], + "interface": "ide2", + "ip_config": [], + "meta_data_file_id": "", + "network_data_file_id": "", + "type": "", + "user_account": [], + "user_data_file_id": "cephfs:snippets/worker4-ignition-user-data", + "vendor_data_file_id": "" + } + ], + "ipv4_addresses": [ + [ + "127.0.0.1" + ], + [ + "10.0.2.104" + ], + [ + "10.0.10.67" + ], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [] + ], + "ipv6_addresses": [ + [ + "::1" + ], + [ + "2001:470:7116:2::104", + "fd00:0:0:2::104", + "fe80::be24:11ff:fe93:db44" + ], + [], + [ + "2001:470:7116:a:ed2a:1bf0:eec6:2100" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ], + [ + "fe80::ecee:eeff:feee:eeee" + ] + ], + "keyboard_layout": "en-us", + "kvm_arguments": null, + "mac_addresses": [ + "00:00:00:00:00:00", + "BC:24:11:93:DB:44", + "00:00:00:00:00:00", + "66:41:F1:FA:94:3B", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE", + "EE:EE:EE:EE:EE:EE" + ], + "machine": null, + "memory": [ + { + "dedicated": 14000, + "floating": 14000, + "hugepages": "", + "keep_hugepages": false, + "shared": 0 + } + ], + "migrate": false, + "name": "worker4", + "network_device": [ + { + "bridge": "vmbr0", + "disconnected": false, + "enabled": true, + "firewall": false, + "mac_address": "BC:24:11:93:DB:44", + "model": "virtio", + "mtu": 0, + "queues": 0, + "rate_limit": 0, + "trunks": "", + "vlan_id": 0 + } + ], + "network_interface_names": [ + "lo", + "eth0", + "tunl0", + "vxlan-v6.calico", + "cali03cb18accdc", + "cali84ee2132997", + "calic8e313ad455", + "cali618b06a08bf", + "calie5d1061f0d2", + "cali74c2c7cab95", + "calid327fb542ec", + "cali57272c14d48", + "cali906311f4498", + "cali2ac9d3e29fd", + "cali6b5a4ca76d8", + "calid0c0262c20d", + "cali191b7b14a78", + "cali473e637c0db", + "calif655aa2ef63", + "cali84d3e8d6131", + "cali82198d56117", + "cali86df6a2a24e", + "califf0314f94ea", + "cali1299561ac78", + "cali12908b5e00f", + "cali482ab78782d", + "calid0b4ea119f4", + "cali17d2589e01b", + "calife77b2f6e7d", + "cali671375f849e", + "cali4c9d0a480a9", + "calicaaec793295", + "cali27e516903c7", + "cali820b67db9a6", + "calia6eb8e5554a", + "calif7837571d88" + ], + "node_name": "pbs", + "numa": [], + "on_boot": true, + "operating_system": [], + "pool_id": null, + "protection": false, + "reboot": false, + "reboot_after_update": true, + "rng": [], + "scsi_hardware": "virtio-scsi-pci", + "serial_device": [], + "smbios": [], + "started": true, + "startup": [ + { + "down_delay": -1, + "order": 1, + "up_delay": -1 + } + ], + "stop_on_destroy": false, + "tablet_device": true, + "tags": [ + "flatcar", + "kubernetes", + "terraform", + "worker" + ], + "template": false, + "timeout_clone": 1800, + "timeout_create": 1800, + "timeout_migrate": 1800, + "timeout_move_disk": 1800, + "timeout_reboot": 1800, + "timeout_shutdown_vm": 1800, + "timeout_start_vm": 1800, + "timeout_stop_vm": 300, + "tpm_state": [], + "usb": [], + "vga": [], + "virtiofs": [], + "vm_id": 1014, + "watchdog": [] + }, + "sensitive_attributes": [], + "identity_schema_version": 0, + "private": "bnVsbA==", + "dependencies": [ + "data.ct_config.control_plane1_ignition", + "data.ct_config.control_plane2_ignition", + "data.ct_config.control_plane3_ignition", + "data.ct_config.worker4_ignition", + "null_resource.wait_for_cluster_ready", + "null_resource.wait_for_cp1_api", + "null_resource.wait_for_cp2_cp3_api", + "proxmox_virtual_environment_download_file.flatcar_image", + "proxmox_virtual_environment_file.control_plane1_ignition", + "proxmox_virtual_environment_file.control_plane2_ignition", + "proxmox_virtual_environment_file.control_plane3_ignition", + "proxmox_virtual_environment_file.worker4_ignition", + "proxmox_virtual_environment_vm.control_plane1", + "proxmox_virtual_environment_vm.control_plane2", + "proxmox_virtual_environment_vm.control_plane3", + "proxmox_virtual_environment_vm.flatcar_template" + ] + } + ] + } + ], "check_results": null }