diff --git a/.DS_Store b/.DS_Store index eb376cc..9766014 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/terraform/main.tf b/terraform/main.tf index 27ffbf1..cce6caa 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -73,6 +73,11 @@ data "ct_config" "worker3_ignition" { strict = false pretty_print = true } +data "ct_config" "worker4_ignition" { + content = file("${path.module}/worker4.bu") + strict = false + pretty_print = true +} resource "proxmox_virtual_environment_file" "control_plane1_ignition" { content_type = "snippets" @@ -128,6 +133,15 @@ resource "proxmox_virtual_environment_file" "worker3_ignition" { file_name = "worker3-ignition-user-data" } } +resource "proxmox_virtual_environment_file" "worker4_ignition" { + content_type = "snippets" + datastore_id = "cephfs" # dein Storage + node_name = "hyper1" # oder einer deiner Cluster-Nodes + source_raw { + data = data.ct_config.worker4_ignition.rendered + file_name = "worker4-ignition-user-data" + } +} @@ -437,3 +451,41 @@ resource "proxmox_virtual_environment_vm" "worker3" { user_data_file_id = "${proxmox_virtual_environment_file.worker3_ignition.id}" } } +esource "proxmox_virtual_environment_vm" "worker4" { + name = "worker4" + node_name = "pbs" + description = "kubernetes worker4" + tags = ["worker","flatcar","kubernetes","terraform"] + depends_on = [ + proxmox_virtual_environment_file.worker3_ignition, + null_resource.wait_for_cp3 + ] + + # Hardware + cpu { + type = "host" + cores = 2 + } + memory { + dedicated = 6000 + floating = 6000 + } + + # Netzwerk + network_device { + bridge = "vmbr0" + model = "virtio" + } + + # Disk (Template klonen oder QCOW2 angeben) + clone { + vm_id = proxmox_virtual_environment_vm.flatcar_template.id + full = true + node_name = "hyper1" + } + + # Ignition-Config + initialization { + user_data_file_id = "${proxmox_virtual_environment_file.worker4_ignition.id}" + } +} \ No newline at end of file diff --git a/terraform/worker4.bu b/terraform/worker4.bu new file mode 100644 index 0000000..770b5ca --- /dev/null +++ b/terraform/worker4.bu @@ -0,0 +1,275 @@ +variant: flatcar +version: 1.1.0 + +passwd: + users: + - name: core + ssh_authorized_keys: + - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHEAlPo3v4U67Y3411pTjIMkQxwlFWdXrBJkSzXenDH flatcar@undercloud" + +storage: + directories: + - path: /opt/bin + overwrite: true + mode: 0755 + - path: /opt/cni/bin + overwrite: true + mode: 0755 + files: + - path: /etc/hostname + mode: 0644 + contents: + inline: | + worker4 + + - path: /etc/systemd/network/00-eth.network + mode: 0644 + contents: + inline: | + [Match] + Name=eth* + + [Network] + Address=fd00:0:0:2::104/64 + Address=2001:470:7116:2::104/64 + Gateway=2001:470:7116:2::3 + DNS=fd00:0:0:1::1 + Address=10.0.2.104/24 + Gateway=10.0.2.3 + DNS=10.0.1.1 + Domains=undercloud.local + IPv6AcceptRA=no + IPv6PrivacyExtensions=no + + - path: /etc/hosts + mode: 0644 + overwrite: true + contents: + inline: | + 127.0.0.1 localhost + ::1 localhost ip6-localhost ip6-loopback + + 2001:470:7116:2::91 control-plane1.undercloud.local control-plane1 + 2001:470:7116:2::92 control-plane2.undercloud.local control-plane2 + 2001:470:7116:2::93 control-plane3.undercloud.local control-plane3 + 2001:470:7116:2::101 worker1.undercloud.local worker1 + 2001:470:7116:2::102 worker2.undercloud.local worker2 + 2001:470:7116:2::103 worker3.undercloud.local worker3 + 2001:470:7116:2::104 worker4.undercloud.local worker4 + + fd00:0:0:2::91 control-plane1.undercloud.local control-plane1 + fd00:0:0:2::92 control-plane2.undercloud.local control-plane2 + fd00:0:0:2::93 control-plane3.undercloud.local control-plane3 + fd00:0:0:2::101 worker1.undercloud.local worker1 + fd00:0:0:2::102 worker2.undercloud.local worker2 + fd00:0:0:2::103 worker3.undercloud.local worker3 + fd00:0:0:2::104 worker4.undercloud.local worker4 + + 10.0.2.91 control-plane1.undercloud.local control-plane1 + 10.0.2.92 control-plane2.undercloud.local control-plane2 + 10.0.2.93 control-plane3.undercloud.local control-plane3 + 10.0.2.101 worker1.undercloud.local worker1 + 10.0.2.102 worker2.undercloud.local worker2 + 10.0.2.103 worker3.undercloud.local worker3 + 10.0.2.104 worker4.undercloud.local worker4 + + - path: /etc/motd + mode: 0644 + overwrite: true + contents: + inline: | + ******************************************************************* + * AUTHORIZED ACCESS ONLY * + * * + * This system is part of a secured infrastructure. * + * All activities are monitored and logged. * + * Unauthorized access or misuse is strictly prohibited and * + * may result in disciplinary and legal action. * + ******************************************************************* + + -------------------------------------------------------------------------------- + kubernetes worker Node + + dont manage + -------------------------------------------------------------------------------- + + - path: /etc/sysctl.d/99-k8s.conf + mode: 0644 + contents: + inline: | + net.ipv4.ip_forward = 1 + net.ipv6.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + net.ipv4.conf.all.forwarding = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.netfilter.nf_conntrack_max = 1000000 + net.ipv4.conf.all.rp_filter = 0 + net.ipv6.conf.all.disable_ipv6 = 0 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 524288 + fs.inotify.max_user_instances = 512 + kernel.panic = 10 + kernel.panic_on_oops = 1 + + - path: /etc/flatcar/update.conf + overwrite: true + mode: 0420 + contents: + inline: | + REBOOT_STRATEGY=off + + - path: /opt/bin/kubeadm + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubeadm" + + - path: /opt/bin/kubelet + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubelet" + + - path: /opt/bin/kubectl + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/kubectl" + + - path: /opt/bin/calicoctl + mode: 0755 + contents: + source: "http://git.undercloud.local:3000/Undercloud/undercloud-infrastructure/raw/branch/main/k8s-binaries/calicoctl" + + - path: /etc/kubernetes/kubeadm-join.yaml + mode: 0644 + contents: + inline: | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: JoinConfiguration + nodeRegistration: + name: worker4 + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + node-ip: "2001:470:7116:2::104" + cluster-dns: "10.0.91.53,2001:470:7116:f:1::53" + volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + discovery: + bootstrapToken: + apiServerEndpoint: "[fd00:0:0:2::100]:6443" + token: "kvg1hc.t3rewovrps426rof" + unsafeSkipCAVerification: true + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + address: "::" + healthzBindAddress: "::" + clusterDomain: "k8s.undercloud.local" + clusterDNS: + - "2001:470:7116:f:1::53" + - "10.0.91.53" + cgroupDriver: "systemd" + volumePluginDir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + +systemd: + units: + - name: modules-load.service + enabled: true + contents: | + [Unit] + Description=Load necessary kernel modules + Before=containerd.service kubeadm-init.service + + [Service] + Type=oneshot + ExecStart=/usr/bin/modprobe br_netfilter + ExecStart=/usr/bin/modprobe overlay + RemainAfterExit=yes + + [Install] + WantedBy=multi-user.target + + - name: systemd-networkd-wait-online.service + enabled: true + + - name: containerd.service + enabled: true + contents: | + [Unit] + Description=containerd container runtime + After=network.target modules-load.service + + [Service] + ExecStart=/usr/bin/containerd + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + OOMScoreAdjust=-999 + + [Install] + WantedBy=multi-user. + + - name: set-timezone.service + enabled: true + contents: | + [Unit] + Description=Set Timezone + After=network-online.target + Wants=network-online.target + [Service] + StandardOutput=journal+console + StandardError=journal+console + Type=oneshot + Restart=on-failure + ExecStart=/usr/bin/timedatectl set-timezone Europe/Berlin + ExecStart=/usr/bin/timedatectl set-ntp true + [Install] + WantedBy=kubeadm.service + + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=kubelet, the Kubernetes Node Agent + Documentation=https://kubernets.io/docs/home + Wants=network-online.target + After=network-online.target + [Service] + #StandardOutput=journal+console + #StandardError=journal+console + #EnvironmentFile=/run/metadata/coreos + Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" + Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" + # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically + EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env + ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + Restart=always + StartLimitInterval=0 + RestartSec=10 + [Install] + WantedBy=multi-user.target + + - name: kubeadm-join.service + enabled: true + contents: | + [Unit] + Description=Join node to Kubernetes cluster + After=network-online.target containerd.service kubelet.service + Wants=network-online.target + + [Service] + Type=oneshot + # Environment + Environment=KUBECONFIG=/etc/kubernetes/admin.conf + Environment=DATASTORE_TYPE=kubernetes + Environment=PATH=/usr/bin/:/usr/sbin:/opt/bin:/opt/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent-uds/ + + ExecStartPre=/bin/sleep 30s + + ExecStart=/opt/bin/kubeadm join --config=/etc/kubernetes/kubeadm-join.yaml + + #ExecStartPost=/usr/bin/systemctl disable kubeadm-init.service + Restart=on-failure + RestartSec=120s + + [Install] + WantedBy=multi-user.target