619 lines
14 KiB
HCL
619 lines
14 KiB
HCL
# ============ Deployment of the K8s cluster ==============
|
|
# = by Sebastian Gurlin for Undercloud =
|
|
# =========================================================
|
|
|
|
terraform {
|
|
required_providers {
|
|
proxmox = {
|
|
source = "bpg/proxmox"
|
|
version = ">=0.35.0"
|
|
}
|
|
ct = {
|
|
source = "poseidon/ct"
|
|
version = ">=0.13.0"
|
|
}
|
|
}
|
|
}
|
|
|
|
provider "proxmox" {
|
|
endpoint = "https://hyper1.undercloud.local:8006/api2/json"
|
|
api_token = "terraform@pve!terraform=5c2ec60a-5605-45cc-bffd-223eb2c487ef"
|
|
insecure = true
|
|
|
|
ssh {
|
|
agent = true
|
|
username = "root"
|
|
}
|
|
}
|
|
|
|
|
|
|
|
# ============ Download Flatcar QCOW ==============
|
|
|
|
resource "proxmox_virtual_environment_download_file" "flatcar_image" {
|
|
content_type = "import"
|
|
datastore_id = "cephfs" # oder dein ISO-Storage
|
|
node_name = "hyper1"
|
|
|
|
url = "http://git.undercloud.local:3000/admin/undercloud-infrastructure/raw/branch/main/iso/flatcar_production_proxmoxve_image.img"
|
|
file_name = "flatcar_production_proxmoxve_image.qcow2" # wird als ISO gespeichert
|
|
}
|
|
|
|
|
|
|
|
# ============== Butane zu Ignition ===============
|
|
|
|
data "ct_config" "control_plane1_ignition" {
|
|
content = file("${path.module}/control-plane1.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "control_plane2_ignition" {
|
|
content = file("${path.module}/control-plane2.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "control_plane3_ignition" {
|
|
content = file("${path.module}/control-plane3.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "worker1_ignition" {
|
|
content = file("${path.module}/worker1.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "worker2_ignition" {
|
|
content = file("${path.module}/worker2.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "worker3_ignition" {
|
|
content = file("${path.module}/worker3.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
data "ct_config" "worker4_ignition" {
|
|
content = file("${path.module}/worker4.bu")
|
|
strict = false
|
|
pretty_print = true
|
|
}
|
|
|
|
resource "proxmox_virtual_environment_file" "control_plane1_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.control_plane1_ignition.rendered
|
|
file_name = "control-plane1-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "control_plane2_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.control_plane2_ignition.rendered
|
|
file_name = "control-plane2-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "control_plane3_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.control_plane3_ignition.rendered
|
|
file_name = "control-plane3-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "worker1_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.worker1_ignition.rendered
|
|
file_name = "worker1-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "worker2_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.worker2_ignition.rendered
|
|
file_name = "worker2-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "worker3_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.worker3_ignition.rendered
|
|
file_name = "worker3-ignition-user-data"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_file" "worker4_ignition" {
|
|
content_type = "snippets"
|
|
datastore_id = "cephfs" # dein Storage
|
|
node_name = "hyper1" # oder einer deiner Cluster-Nodes
|
|
source_raw {
|
|
data = data.ct_config.worker4_ignition.rendered
|
|
file_name = "worker4-ignition-user-data"
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
# ============== flatcar template anlegen ================
|
|
|
|
resource "proxmox_virtual_environment_vm" "flatcar_template" {
|
|
name = "flatcar-template"
|
|
node_name = "hyper1"
|
|
template = true
|
|
started = false
|
|
stop_on_destroy = true
|
|
description = "managed by terraform - base template for flatcar"
|
|
tags = ["flatcar","kubernetes","terraform"]
|
|
vm_id = 999
|
|
|
|
lifecycle {
|
|
ignore_changes = [
|
|
boot_order,
|
|
disk, # ignore disk drift like ssd flag
|
|
]
|
|
}
|
|
|
|
agent {
|
|
enabled = true
|
|
}
|
|
|
|
cpu {
|
|
type = "host"
|
|
cores = 1
|
|
}
|
|
|
|
memory {
|
|
dedicated = 2048
|
|
floating = 2048
|
|
}
|
|
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Bootdisk direkt aus dem QCOW2
|
|
disk {
|
|
datastore_id = "Pool1"
|
|
import_from = proxmox_virtual_environment_download_file.flatcar_image.id
|
|
interface = "virtio0"
|
|
discard = "on"
|
|
size = 50
|
|
cache = "writeback"
|
|
ssd = true
|
|
}
|
|
|
|
# Cloud-Init Laufwerk hinzufügen
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
}
|
|
|
|
|
|
boot_order = ["virtio0", "ide2"]
|
|
}
|
|
|
|
resource "null_resource" "wait_for_cp1_api" {
|
|
depends_on = [proxmox_virtual_environment_vm.control_plane1]
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
environment = {
|
|
CP1 = "control-plane1.undercloud.local" # <-- set this
|
|
}
|
|
|
|
command = <<EOF
|
|
set -euo pipefail
|
|
|
|
deadline=$((SECONDS + 1800)) # 30 min
|
|
echo "Waiting for Kubernetes API on $CP1:6443 ..."
|
|
|
|
until curl -kfsS "https://$CP1:6443/healthz" | grep -qx "ok"; do
|
|
if [ $SECONDS -ge $deadline ]; then
|
|
echo "Timeout waiting for API server"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
echo "API server is up."
|
|
EOF
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "wait_for_cp2_cp3_api" {
|
|
depends_on = [
|
|
proxmox_virtual_environment_vm.control_plane2,
|
|
proxmox_virtual_environment_vm.control_plane3,
|
|
]
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
environment = {
|
|
CP2 = "control-plane2.undercloud.local" # set this
|
|
CP3 = "control-plane3.undercloud.local" # set this
|
|
}
|
|
|
|
command = <<EOF
|
|
set -euo pipefail
|
|
|
|
deadline=$((SECONDS + 2400)) # 40 min
|
|
|
|
check() {
|
|
local ip="$1"
|
|
curl -kfsS "https://$ip:6443/healthz" | grep -qx "ok"
|
|
}
|
|
|
|
echo "Waiting for cp2 API..."
|
|
until check "$CP2"; do
|
|
if [ $SECONDS -ge $deadline ]; then echo "Timeout waiting for cp2 API"; exit 1; fi
|
|
sleep 5
|
|
done
|
|
echo "cp2 API is up."
|
|
|
|
echo "Waiting for cp3 API..."
|
|
until check "$CP3"; do
|
|
if [ $SECONDS -ge $deadline ]; then echo "Timeout waiting for cp3 API"; exit 1; fi
|
|
sleep 5
|
|
done
|
|
echo "cp3 API is up."
|
|
EOF
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "wait_for_cluster_ready" {
|
|
depends_on = [
|
|
null_resource.wait_for_cp1_api,
|
|
null_resource.wait_for_cp2_cp3_api,
|
|
]
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
environment = {
|
|
CP1_HOST = "control-plane1.undercloud.local"
|
|
}
|
|
|
|
command = <<EOF
|
|
set -euo pipefail
|
|
deadline=$((SECONDS + 1800))
|
|
|
|
echo "Waiting for cluster readiness on $CP1_HOST (/readyz) ..."
|
|
until curl -kfsS "https://$CP1_HOST:6443/readyz" | grep -qx "ok"; do
|
|
if [ $SECONDS -ge $deadline ]; then
|
|
echo "Timeout waiting for /readyz"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
echo "Cluster API reports ready."
|
|
EOF
|
|
}
|
|
}
|
|
|
|
# =============== deploy Control Plane ====================
|
|
|
|
resource "proxmox_virtual_environment_vm" "control_plane1" {
|
|
name = "control-plane1"
|
|
node_name = "hyper1"
|
|
description = "kubernetes control-plane1"
|
|
tags = ["control-plane","flatcar","kubernetes","terraform"]
|
|
depends_on = [proxmox_virtual_environment_file.control_plane1_ignition]
|
|
boot_order = ["virtio0"]
|
|
vm_id = 1001
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 4
|
|
}
|
|
memory {
|
|
dedicated = 4096
|
|
floating = 4096
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.control_plane1_ignition.id}"
|
|
}
|
|
}
|
|
|
|
resource "proxmox_virtual_environment_vm" "control_plane2" {
|
|
name = "control-plane2"
|
|
vm_id = 1002
|
|
node_name = "hyper2"
|
|
description = "kubernetes control-plane2"
|
|
tags = ["control-plane","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.control_plane2_ignition,
|
|
null_resource.wait_for_cp1_api
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 4
|
|
}
|
|
memory {
|
|
dedicated = 4096
|
|
floating = 4096
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.control_plane2_ignition.id}"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_vm" "control_plane3" {
|
|
name = "control-plane3"
|
|
vm_id = 1003
|
|
node_name = "hyper3"
|
|
description = "kubernetes control-plane3"
|
|
tags = ["control-plane","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.control_plane3_ignition,
|
|
null_resource.wait_for_cp1_api
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 4
|
|
}
|
|
memory {
|
|
dedicated = 4096
|
|
floating = 4096
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.control_plane3_ignition.id}"
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
# =============== deploy Workers ====================
|
|
|
|
resource "proxmox_virtual_environment_vm" "worker1" {
|
|
name = "worker1"
|
|
vm_id = 1011
|
|
node_name = "hyper3"
|
|
description = "kubernetes worker1"
|
|
tags = ["worker","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.worker1_ignition,
|
|
null_resource.wait_for_cluster_ready
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 6
|
|
}
|
|
memory {
|
|
dedicated = 25000
|
|
floating = 25000
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.worker1_ignition.id}"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_vm" "worker2" {
|
|
name = "worker2"
|
|
vm_id = 1012
|
|
node_name = "hyper4"
|
|
description = "kubernetes worker2"
|
|
tags = ["worker","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.worker2_ignition,
|
|
null_resource.wait_for_cluster_ready
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 14
|
|
}
|
|
memory {
|
|
dedicated = 14000
|
|
floating = 14000
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.worker2_ignition.id}"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_vm" "worker3" {
|
|
name = "worker3"
|
|
vm_id = 1013
|
|
node_name = "hyper6"
|
|
description = "kubernetes worker3"
|
|
tags = ["worker","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.worker3_ignition,
|
|
null_resource.wait_for_cluster_ready
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 6
|
|
}
|
|
memory {
|
|
dedicated = 14000
|
|
floating = 14000
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.worker3_ignition.id}"
|
|
}
|
|
}
|
|
resource "proxmox_virtual_environment_vm" "worker4" {
|
|
name = "worker4"
|
|
vm_id = 1014
|
|
node_name = "pbs"
|
|
description = "kubernetes worker4"
|
|
tags = ["worker","flatcar","kubernetes","terraform"]
|
|
depends_on = [
|
|
proxmox_virtual_environment_file.worker4_ignition,
|
|
null_resource.wait_for_cluster_ready
|
|
]
|
|
boot_order = ["virtio0"]
|
|
|
|
startup {
|
|
order = "1"
|
|
}
|
|
# Hardware
|
|
cpu {
|
|
type = "host"
|
|
cores = 4
|
|
}
|
|
memory {
|
|
dedicated = 14000
|
|
floating = 14000
|
|
}
|
|
|
|
# Netzwerk
|
|
network_device {
|
|
bridge = "vmbr0"
|
|
model = "virtio"
|
|
}
|
|
|
|
# Disk (Template klonen oder QCOW2 angeben)
|
|
clone {
|
|
vm_id = proxmox_virtual_environment_vm.flatcar_template.id
|
|
full = true
|
|
node_name = "hyper1"
|
|
}
|
|
|
|
# Ignition-Config
|
|
initialization {
|
|
datastore_id = "Pool1"
|
|
user_data_file_id = "${proxmox_virtual_environment_file.worker4_ignition.id}"
|
|
}
|
|
}
|