feat: Declarative K3S node

flake.lock

@@ -218,11 +218,11 @@
     },
     "secrets": {
       "locked": {
-        "lastModified": 1764605254,
+        "lastModified": 1766822527,
-        "narHash": "sha256-xpKD2OS2YgqxmIzuPU07OYtOqp6pccKdamb+oXhch7w=",
+        "narHash": "sha256-0qNxAxr7LQ8C6MjSxV2FkMSfdVmOVRq7Yz/wCea8plo=",
         "ref": "refs/heads/main",
-        "rev": "c1bba633c0006784e95d9c10e937a15ea3aaee55",
+        "rev": "695e36891b5de248993845e1435df5e4116a26f2",
-        "revCount": 20,
+        "revCount": 21,
         "type": "git",
         "url": "ssh://gitea@git.bulthuis.dev/Jan/nixos-secrets"
       },

hosts/vm-k1s/configuration.nix

@@ -0,0 +1,168 @@
+{
+  inputs,
+  pkgs,
+  config,
+  ...
+}:
+
+{
+  # State version
+  system.stateVersion = "25.05";
+
+  # Machine hostname
+  networking.hostName = "vm-k1s";
+
+  # Enabled modules
+  modules = {
+    profiles.vm.enable = true;
+  };
+
+  # Read in secrets
+  sops.secrets."flux/git-ssh-key" = {
+    sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml";
+  };
+  sops.secrets."flux/sops-decrypt-key" = {
+    sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml";
+  };
+
+  # Include NFS client module
+  boot.supportedFilesystems = [ "nfs" ];
+
+  # Set up K3S cluster with CoreDNS and FluxCD
+  services.k3s = {
+    enable = true;
+    extraFlags = [
+      "--cluster-domain ${inputs.secrets.lab.k3s.clusterDomain}"
+    ];
+    disable = [
+      # "coredns" # CoreDNS is required for Flux to be able to bootstrap the cluster (Flux needs to resolve the git repo)
+      # "servicelb" # Required for Traefik, can later be replaced with load balancer deployed through Flux
+      "traefik"
+      "local-storage"
+      "metrics-server"
+      "runtimes"
+    ];
+    manifests = {
+      git-ssh-key = {
+        source = config.sops.secrets."flux/git-ssh-key".path;
+      };
+      sops-decrypt-key = {
+        source = config.sops.secrets."flux/sops-decrypt-key".path;
+      };
+      "0-secrets-backup-namespaces" = {
+        source = "/opt/k3s-secrets-backup/namespaces.yaml";
+      };
+      "1-secrets-backup" = {
+        source = "/opt/k3s-secrets-backup/secrets.yaml";
+      };
+    };
+    autoDeployCharts = {
+      flux-operator = {
+        name = "flux-operator";
+        repo = "oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator";
+        version = "0.38.1";
+        hash = "sha256-nb0mzEWC3IwjPenQ4LSWBN0NNJc2cc68RB+G60xBOEM=";
+        createNamespace = true;
+        targetNamespace = "flux-system";
+        extraDeploy = [
+          {
+            apiVersion = "fluxcd.controlplane.io/v1";
+            kind = "FluxInstance";
+            metadata = {
+              name = "flux";
+              namespace = "flux-system";
+              annotations = {
+                "fluxcd.controlplane.io/reconcile" = "enabled";
+                "fluxcd.controlplane.io/reconcileEvery" = "1h";
+                "fluxcd.controlplane.io/reconcileTimeout" = "5m";
+              };
+            };
+            spec = {
+              distribution = {
+                version = "2.x";
+                registry = "ghcr.io/fluxcd";
+              };
+              components = [
+                "source-controller"
+                "kustomize-controller"
+                "helm-controller"
+                "notification-controller"
+              ];
+              cluster = {
+                type = "kubernetes";
+                size = "small";
+                multitenant = false;
+                networkPolicy = true;
+                domain = inputs.secrets.lab.k3s.clusterDomain;
+              };
+              commonMetadata.labels = {
+                "app.kubernetes.io/name" = "flux";
+              };
+              sync = (
+                {
+                  pullSecret = "git-ssh-key";
+                }
+                // inputs.secrets.lab.k3s.fluxRepo
+              );
+            };
+          }
+        ];
+      };
+    };
+  };
+
+  # Backup secrets to avoid reissueing them
+  modules.impermanence.directories = [
+    "/opt/k3s-secrets-backup"
+  ];
+  systemd.timers.k3s-secrets-backup-timer = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = "15m";
+      OnUnitActiveSec = "1h";
+      Unit = "k3s-secrets-backup.service";
+    };
+  };
+  systemd.services.k3s-secrets-backup = {
+    script = ''
+      mkdir -p /opt/k3s-secrets-backup
+      touch /opt/k3s-secrets-backup/secrets.yaml
+      touch /opt/k3s-secrets-backup/namespaces.yaml
+      chmod 600 /opt/k3s-secrets-backup/secrets.yaml
+      chmod 600 /opt/k3s-secrets-backup/namespaces.yaml
+
+      ${pkgs.k3s}/bin/kubectl get secrets -A -l controller.cert-manager\.io/fao=="true" -oyaml | ${pkgs.kubectl-neat}/bin/kubectl-neat > /opt/k3s-secrets-backup/secrets.yaml
+
+      echo "apiVersion: v1
+      kind: List
+      items:" > /opt/k3s-secrets-backup/namespaces.yaml
+
+      ${pkgs.gnugrep}/bin/grep -oP '\snamespace: \K.*' /opt/k3s-secrets-backup/secrets.yaml | sort -u | while read -r ns; do
+        echo "- apiVersion: v1
+        kind: Namespace
+        metadata:
+          name: $ns"
+      done >> /opt/k3s-secrets-backup/namespaces.yaml
+    '';
+    serviceConfig = {
+      Type = "oneshot";
+      User = "root";
+    };
+  };
+
+  environment.variables = {
+    KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
+  };
+
+  environment.systemPackages = with pkgs; [
+    fluxcd
+    k9s
+  ];
+
+  # Use correct disko profile
+  modules.disko.profile = "k3s";
+
+  # TEMP: Disable firewall for now
+  networking.firewall.enable = false;
+  security.sudo.wheelNeedsPassword = false;
+}

profiles/disko/k3s.nix

@@ -0,0 +1,82 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/sda";
+        imageSize = "32G"; # For test VMs
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            zfs = {
+              end = "-4G";
+              content = {
+                type = "zfs";
+                pool = "tank";
+              };
+            };
+            swap = {
+              size = "100%";
+              content = {
+                type = "swap";
+                discardPolicy = "both";
+              };
+            };
+          };
+        };
+      };
+      longhorn = {
+        type = "disk";
+        device = "/dev/sdb";
+        imageSize = "64G"; # For longhorn storage
+        content = {
+          type = "gpt";
+          partitions = {
+            main = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      tank = {
+        type = "zpool";
+        rootFsOptions = {
+          compression = "zstd";
+        };
+        mountpoint = null;
+        postCreateHook = "zfs snapshot -r tank@blank && zfs hold -r blank tank@blank";
+
+        datasets = {
+          root = {
+            type = "zfs_fs";
+            mountpoint = "/";
+          };
+          nix = {
+            type = "zfs_fs";
+            mountpoint = "/nix";
+          };
+          persist = {
+            type = "zfs_fs";
+            mountpoint = "/persist";
+          };
+        };
+      };
+    };
+  };
+}

profiles/nixos/base.nix

@@ -75,6 +75,7 @@ in
       zip
       unzip
       tmux
+      unixtools.net-tools
     ];
   };
 }

profiles/nixos/vm.nix

@@ -20,7 +20,7 @@ in
       profiles.base.enable = true;
       disko = {
         enable = true;
-        profile = "vm";
+        profile = mkDefault "vm";
       };
       impermanence = {
         enable = true;