nixos-config/hosts/vm-k1s/configuration.nix

{
  inputs,
  pkgs,
  config,
  ...
}:

{
  # State version
  system.stateVersion = "25.05";

  # Machine hostname
  networking.hostName = "vm-k1s";

  # Enabled modules
  modules = {
    profiles.vm.enable = true;
  };

  # Read in secrets
  sops.secrets."flux/git-ssh-key" = {
    sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml";
  };
  sops.secrets."flux/sops-decrypt-key" = {
    sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml";
  };

  # Include NFS client module
  boot.supportedFilesystems = [ "nfs" ];

  # Set up K3S cluster with CoreDNS and FluxCD
  services.k3s = {
    enable = true;
    extraFlags = [
      "--cluster-domain ${inputs.secrets.lab.k3s.clusterDomain}"
      "--flannel-backend=none"
      "--disable-network-policy"
      "--disable-kube-proxy"
    ];
    disable = [
      # "coredns" # CoreDNS is required for Flux to be able to bootstrap the cluster (Flux needs to resolve the git repo)
      "servicelb"
      "traefik"
      "local-storage"
      "metrics-server"
      "runtimes"
    ];
    manifests = {
      git-ssh-key = {
        source = config.sops.secrets."flux/git-ssh-key".path;
      };
      sops-decrypt-key = {
        source = config.sops.secrets."flux/sops-decrypt-key".path;
      };
      # "0-secrets-backup-namespaces" = {
      #   source = "/opt/k3s-secrets-backup/namespaces.yaml";
      # };
      # "1-secrets-backup" = {
      #   source = "/opt/k3s-secrets-backup/secrets.yaml";
      # };
      # TODO: Move to flux config, once it is possible to easily install flux without CNI
      cilium-secrets-namespace = {
        content = {
          apiVersion = "v1";
          kind = "Namespace";
          metadata.name = "cilium-secrets";
        };
      };
      # TODO: Move to flux config, once it is possible to easily install flux without CNI
      gateway-api =
        let
          manifest = pkgs.fetchurl {
            url = "https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/experimental-install.yaml";
            hash = "sha256-08IN1MBDGTZWemkXypMfbc7RMQJCvmK57KB72YkuICU=";
          };
        in
        {
          source = manifest;
        };
      # TODO: Move to flux config, once it is possible to easily install flux without CNI
      netpol-system-allow-egress.content = {
        apiVersion = "cilium.io/v2";
        kind = "CiliumClusterwideNetworkPolicy";
        metadata.name = "allow-system-egress";
        spec = {
          description = "Allow all egress to system services.";
          endpointSelector = {
            matchExpressions = [
              {
                key = "io.kubernetes.pod.namespace";
                operator = "NotIn";
                values = [
                  "bogus-namespace"
                  # "kube-system"
                  # "cilium-system"
                  # "flux-system"
                ];
              }
            ];
          };
          egress = [
            {
              toEntities = [
                "all"
              ];
            }
          ];
          ingress = [
            {
              fromEntities = [
                "all"
              ];
            }
          ];
        };
      };
      netpol-cluster-allow-dns.content = {
        apiVersion = "cilium.io/v2";
        kind = "CiliumClusterwideNetworkPolicy";
        metadata.name = "allow-dns";
        spec = {
          description = "Allow DNS";
          endpointSelector = { };
          egress = [
            {
              toEndpoints = [
                {
                  matchLabels = {
                    "io.kubernetes.pod.namespace" = "kube-system";
                    "k8s-app" = "kube-dns";
                  };
                }
              ];
              toPorts = [
                {
                  ports = [
                    {
                      port = 53;
                      protocol = "ANY";
                    }
                  ];
                  rules.dns = [
                    {
                      matchPattern = "*";
                    }
                  ];
                }
              ];
            }
          ];
        };
      };
      netpol-flux-allow-egress.content = { };
    };
    autoDeployCharts = {
      # TODO: Move to flux config, once it is possible to easily install flux without CNI
      cilium = {
        name = "cilium";
        repo = "oci://quay.io/cilium/charts/cilium";
        version = "1.18.6";
        hash = "sha256-+yr38lc5X1+eXCFE/rq/K0m4g/IiNFJHuhB+Nu24eUs=";
        createNamespace = true;
        targetNamespace = "cilium-system";
        values = {
          operator.replicas = 1;
          kubeProxyReplacement = true;
          ipam.operator.clusterPoolIPv4PodCIDRList = [ "10.42.0.0/16" ];
          cluster = {
            id = 1;
            name = "vm-k1s";
          };
          k8sServiceHost = "10.10.50.60";
          k8sServicePort = 6443;
          policyEnforcementMode = "always";
          gatewayAPI = {
            enabled = true;
            gatewayClass.create = "true";
            secretsNamespace.create = false;
            enableAlpn = true;
          };
          bgpControlPlane.enabled = true;
          tls.secretsNamespace.create = false;
          hubble = {
            relay.enabled = true;
            ui.enabled = true;
            peerService.clusterDomain = inputs.secrets.lab.k3s.clusterDomain;
          };
        };
        extraFieldDefinitions = {
          spec.bootstrap = true;
        };
      };
      flux-operator = {
        name = "flux-operator";
        repo = "oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator";
        version = "0.38.1";
        hash = "sha256-nb0mzEWC3IwjPenQ4LSWBN0NNJc2cc68RB+G60xBOEM=";
        createNamespace = true;
        targetNamespace = "flux-system";
        extraDeploy = [
          {
            apiVersion = "fluxcd.controlplane.io/v1";
            kind = "FluxInstance";
            metadata = {
              name = "flux";
              namespace = "flux-system";
              annotations = {
                "fluxcd.controlplane.io/reconcile" = "enabled";
                "fluxcd.controlplane.io/reconcileEvery" = "1h";
                "fluxcd.controlplane.io/reconcileTimeout" = "5m";
              };
            };
            spec = {
              distribution = {
                version = "2.x";
                registry = "ghcr.io/fluxcd";
              };
              components = [
                "source-controller"
                "kustomize-controller"
                "helm-controller"
                "notification-controller"
              ];
              cluster = {
                type = "kubernetes";
                size = "small";
                multitenant = false;
                networkPolicy = true;
                domain = inputs.secrets.lab.k3s.clusterDomain;
              };
              commonMetadata.labels = {
                "app.kubernetes.io/name" = "flux";
              };
              sync = (
                {
                  pullSecret = "git-ssh-key";
                }
                // inputs.secrets.lab.k3s.fluxRepo
              );
            };
          }
        ];
      };
    };
  };

  # Backup secrets to avoid reissueing them
  modules.impermanence.directories = [
    "/opt/k3s-secrets-backup"
  ];
  systemd.timers.k3s-secrets-backup-timer = {
    wantedBy = [ "timers.target" ];
    timerConfig = {
      OnBootSec = "15m";
      OnUnitActiveSec = "1h";
      Unit = "k3s-secrets-backup.service";
    };
  };
  systemd.services.k3s-secrets-backup = {
    script = ''
      mkdir -p /opt/k3s-secrets-backup
      touch /opt/k3s-secrets-backup/secrets.yaml
      touch /opt/k3s-secrets-backup/namespaces.yaml
      chmod 600 /opt/k3s-secrets-backup/secrets.yaml
      chmod 600 /opt/k3s-secrets-backup/namespaces.yaml

      ${pkgs.k3s}/bin/kubectl get secrets -A -l controller.cert-manager\.io/fao=="true" -oyaml | ${pkgs.kubectl-neat}/bin/kubectl-neat > /opt/k3s-secrets-backup/secrets.yaml

      echo "apiVersion: v1
      kind: List
      items:" > /opt/k3s-secrets-backup/namespaces.yaml

      ${pkgs.gnugrep}/bin/grep -oP '\snamespace: \K.*' /opt/k3s-secrets-backup/secrets.yaml | sort -u | while read -r ns; do
        echo "- apiVersion: v1
        kind: Namespace
        metadata:
          name: $ns"
      done >> /opt/k3s-secrets-backup/namespaces.yaml
    '';
    serviceConfig = {
      Type = "oneshot";
      User = "root";
    };
  };

  environment.variables = {
    KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
    CILIUM_NAMESPACE = "cilium-system";
  };

  environment.systemPackages = with pkgs; [
    fluxcd
    k9s
    cilium-cli
    hubble
  ];

  # Use correct disko profile
  modules.disko.profile = "k3s";

  # TEMP: Disable firewall for now
  networking.firewall.enable = false;
  security.sudo.wheelNeedsPassword = false;
}