# ============ Deployment of the K8s cluster ==============
# =         by Sebastian Gurlin for Undercloud            =
# =========================================================

terraform {
  required_providers {
    proxmox = {
      source  = "bpg/proxmox"
      version = ">=0.35.0"
    }
    ct = {
      source  = "poseidon/ct"
      version = ">=0.13.0"
    }
  }
}

provider "proxmox" {
  endpoint  = "https://hyper1.undercloud.local:8006/api2/json"
  api_token = "terraform@pve!terraform=5c2ec60a-5605-45cc-bffd-223eb2c487ef"
  insecure  = true
  
  ssh {
    agent = true
    username = "root"
  }
}



# ============ Download Flatcar QCOW ==============

resource "proxmox_virtual_environment_download_file" "flatcar_image" {
  content_type = "import"
  datastore_id = "cephfs"            # oder dein ISO-Storage
  node_name    = "hyper1"

  url = "http://git.undercloud.local:3000/admin/undercloud-infrastructure/raw/branch/main/iso/flatcar_production_proxmoxve_image.img"
  file_name = "flatcar_production_proxmoxve_image.qcow2"  # wird als ISO gespeichert
}



# ============== Butane zu Ignition ===============

data "ct_config" "control_plane1_ignition" {
  content       = file("${path.module}/control-plane1.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "control_plane2_ignition" {
  content       = file("${path.module}/control-plane2.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "control_plane3_ignition" {
  content       = file("${path.module}/control-plane3.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "worker1_ignition" {
  content       = file("${path.module}/worker1.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "worker2_ignition" {
  content       = file("${path.module}/worker2.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "worker3_ignition" {
  content       = file("${path.module}/worker3.bu")
  strict        = false
  pretty_print  = true
}
data "ct_config" "worker4_ignition" {
  content       = file("${path.module}/worker4.bu")
  strict        = false
  pretty_print  = true
}

resource "proxmox_virtual_environment_file" "control_plane1_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.control_plane1_ignition.rendered
    file_name    = "control-plane1-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "control_plane2_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.control_plane2_ignition.rendered
    file_name    = "control-plane2-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "control_plane3_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.control_plane3_ignition.rendered
    file_name    = "control-plane3-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "worker1_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.worker1_ignition.rendered
    file_name    = "worker1-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "worker2_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.worker2_ignition.rendered
    file_name    = "worker2-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "worker3_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.worker3_ignition.rendered
    file_name    = "worker3-ignition-user-data"
  }
}
resource "proxmox_virtual_environment_file" "worker4_ignition" {
  content_type = "snippets"
  datastore_id = "cephfs"                 # dein Storage
  node_name    = "hyper1"                 # oder einer deiner Cluster-Nodes
  source_raw {
    data = data.ct_config.worker4_ignition.rendered
    file_name    = "worker4-ignition-user-data"
  }
}




# ============== flatcar template anlegen ================

resource "proxmox_virtual_environment_vm" "flatcar_template" {
  name            = "flatcar-template"
  node_name       = "hyper1"
  template        = true
  started         = false
  stop_on_destroy = true
  description     = "managed by terraform - base template for flatcar"
  tags            = ["flatcar","kubernetes","terraform"]
  
  lifecycle {
    ignore_changes = [
      boot_order,
      disk,         # ignore disk drift like ssd flag
    ]
  }
  
  agent {
    enabled = true
  }

  cpu {
    type  = "host"
    cores = 1
  }

  memory {
    dedicated = 2048
    floating  = 2048
  }

  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Bootdisk direkt aus dem QCOW2
  disk {
    datastore_id = "Pool1"
    import_from  = proxmox_virtual_environment_download_file.flatcar_image.id
    interface    = "virtio0"
    discard      = "on"
    size         = 50
    cache        = "writeback"
    ssd          = true
  }

  # Cloud-Init Laufwerk hinzufügen
  initialization {
    datastore_id = "Pool1"
  }


  boot_order = ["virtio0", "ide2"]
}

resource "null_resource" "wait_for_cp1_api" {
  depends_on = [proxmox_virtual_environment_vm.control_plane1]

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    environment = {
      CP1 = "control-plane1.undercloud.local" # <-- set this
    }

    command = <<EOF
set -euo pipefail

deadline=$((SECONDS + 1800))  # 30 min
echo "Waiting for Kubernetes API on $CP1:6443 ..."

until curl -kfsS "https://$CP1:6443/healthz" | grep -qx "ok"; do
  if [ $SECONDS -ge $deadline ]; then
    echo "Timeout waiting for API server"
    exit 1
  fi
  sleep 5
done

echo "API server is up."
EOF
  }
}

resource "null_resource" "wait_for_cp2_cp3_api" {
  depends_on = [
    proxmox_virtual_environment_vm.control_plane2,
    proxmox_virtual_environment_vm.control_plane3,
  ]

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    environment = {
      CP2 = "control-plane2.undercloud.local" # set this
      CP3 = "control-plane3.undercloud.local" # set this
    }

    command = <<EOF
set -euo pipefail

deadline=$((SECONDS + 2400)) # 40 min

check() {
  local ip="$1"
  curl -kfsS "https://$ip:6443/healthz" | grep -qx "ok"
}

echo "Waiting for cp2 API..."
until check "$CP2"; do
  if [ $SECONDS -ge $deadline ]; then echo "Timeout waiting for cp2 API"; exit 1; fi
  sleep 5
done
echo "cp2 API is up."

echo "Waiting for cp3 API..."
until check "$CP3"; do
  if [ $SECONDS -ge $deadline ]; then echo "Timeout waiting for cp3 API"; exit 1; fi
  sleep 5
done
echo "cp3 API is up."
EOF
  }
}

resource "null_resource" "wait_for_cluster_ready" {
  depends_on = [
    null_resource.wait_for_cp1_api,
    null_resource.wait_for_cp2_cp3_api,
  ]

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    environment = {
      CP1_HOST = "control-plane1.undercloud.local"
    }

    command = <<EOF
set -euo pipefail
deadline=$((SECONDS + 1800))

echo "Waiting for cluster readiness on $CP1_HOST (/readyz) ..."
until curl -kfsS "https://$CP1_HOST:6443/readyz" | grep -qx "ok"; do
  if [ $SECONDS -ge $deadline ]; then
    echo "Timeout waiting for /readyz"
    exit 1
  fi
  sleep 5
done
echo "Cluster API reports ready."
EOF
  }
}

# =============== deploy Control Plane ====================

resource "proxmox_virtual_environment_vm" "control_plane1" {
  name        = "control-plane1"
  node_name   = "hyper1"
  description = "kubernetes control-plane1"
  tags        = ["control-plane","flatcar","kubernetes","terraform"]
  depends_on  = [proxmox_virtual_environment_file.control_plane1_ignition]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }

  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 4096
    floating  = 4096
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.control_plane1_ignition.id}"
  }
}

resource "proxmox_virtual_environment_vm" "control_plane2" {
  name        = "control-plane2"
  node_name   = "hyper2"
  description = "kubernetes control-plane2"
  tags        = ["control-plane","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.control_plane2_ignition,
    null_resource.wait_for_cp1_api
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 3072
    floating  = 3072
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.control_plane2_ignition.id}"
  }
}
resource "proxmox_virtual_environment_vm" "control_plane3" {
  name        = "control-plane3"
  node_name   = "hyper3"
  description = "kubernetes control-plane3"
  tags        = ["control-plane","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.control_plane3_ignition,
    null_resource.wait_for_cp1_api
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 3072
    floating  = 3072
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.control_plane3_ignition.id}"
  }
}




# =============== deploy Workers ====================

resource "proxmox_virtual_environment_vm" "worker1" {
  name        = "worker1"
  node_name   = "hyper1"
  description = "kubernetes worker1"
  tags        = ["worker","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.worker1_ignition,
    null_resource.wait_for_cluster_ready
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 10240
    floating  = 10240
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.worker1_ignition.id}"
  }
}
resource "proxmox_virtual_environment_vm" "worker2" {
  name        = "worker2"
  node_name   = "hyper2"
  description = "kubernetes worker2"
  tags        = ["worker","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.worker2_ignition,
    null_resource.wait_for_cluster_ready
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 10240
    floating  = 10240
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.worker2_ignition.id}"
  }
}
resource "proxmox_virtual_environment_vm" "worker3" {
  name        = "worker3"
  node_name   = "hyper3"
  description = "kubernetes worker3"
  tags        = ["worker","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.worker3_ignition,
    null_resource.wait_for_cluster_ready
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 10240
    floating  = 10240
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.worker3_ignition.id}"
  }
}
resource "proxmox_virtual_environment_vm" "worker4" {
  name        = "worker4"
  node_name   = "pbs"
  description = "kubernetes worker4"
  tags        = ["worker","flatcar","kubernetes","terraform"]
  depends_on  = [
    proxmox_virtual_environment_file.worker4_ignition,
    null_resource.wait_for_cluster_ready
  ]
  boot_order = ["virtio0"]

  startup {
    order      = "1"
  }
  # Hardware
  cpu {
    type  = "host"
    cores = 2
  }
  memory {
    dedicated = 12000
    floating  = 12000
  }

  # Netzwerk
  network_device {
    bridge = "vmbr0"
    model  = "virtio"
  }

  # Disk (Template klonen oder QCOW2 angeben)
  clone {
    vm_id     = proxmox_virtual_environment_vm.flatcar_template.id
    full      = true
    node_name = "hyper1"
  }

  # Ignition-Config
  initialization {
    datastore_id = "Pool1"
    user_data_file_id = "${proxmox_virtual_environment_file.worker4_ignition.id}"
  }
}