diff --git a/flake.lock b/flake.lock index 5048f0f..0603d69 100644 --- a/flake.lock +++ b/flake.lock @@ -218,11 +218,11 @@ }, "secrets": { "locked": { - "lastModified": 1764605254, - "narHash": "sha256-xpKD2OS2YgqxmIzuPU07OYtOqp6pccKdamb+oXhch7w=", + "lastModified": 1766822527, + "narHash": "sha256-0qNxAxr7LQ8C6MjSxV2FkMSfdVmOVRq7Yz/wCea8plo=", "ref": "refs/heads/main", - "rev": "c1bba633c0006784e95d9c10e937a15ea3aaee55", - "revCount": 20, + "rev": "695e36891b5de248993845e1435df5e4116a26f2", + "revCount": 21, "type": "git", "url": "ssh://gitea@git.bulthuis.dev/Jan/nixos-secrets" }, diff --git a/hosts/vm-k1s/configuration.nix b/hosts/vm-k1s/configuration.nix new file mode 100644 index 0000000..474edf3 --- /dev/null +++ b/hosts/vm-k1s/configuration.nix @@ -0,0 +1,168 @@ +{ + inputs, + pkgs, + config, + ... +}: + +{ + # State version + system.stateVersion = "25.05"; + + # Machine hostname + networking.hostName = "vm-k1s"; + + # Enabled modules + modules = { + profiles.vm.enable = true; + }; + + # Read in secrets + sops.secrets."flux/git-ssh-key" = { + sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml"; + }; + sops.secrets."flux/sops-decrypt-key" = { + sopsFile = "${inputs.secrets}/secrets/k3s-cluster.enc.yaml"; + }; + + # Include NFS client module + boot.supportedFilesystems = [ "nfs" ]; + + # Set up K3S cluster with CoreDNS and FluxCD + services.k3s = { + enable = true; + extraFlags = [ + "--cluster-domain ${inputs.secrets.lab.k3s.clusterDomain}" + ]; + disable = [ + # "coredns" # CoreDNS is required for Flux to be able to bootstrap the cluster (Flux needs to resolve the git repo) + # "servicelb" # Required for Traefik, can later be replaced with load balancer deployed through Flux + "traefik" + "local-storage" + "metrics-server" + "runtimes" + ]; + manifests = { + git-ssh-key = { + source = config.sops.secrets."flux/git-ssh-key".path; + }; + sops-decrypt-key = { + source = config.sops.secrets."flux/sops-decrypt-key".path; + }; + "0-secrets-backup-namespaces" = { + source = "/opt/k3s-secrets-backup/namespaces.yaml"; + }; + "1-secrets-backup" = { + source = "/opt/k3s-secrets-backup/secrets.yaml"; + }; + }; + autoDeployCharts = { + flux-operator = { + name = "flux-operator"; + repo = "oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator"; + version = "0.38.1"; + hash = "sha256-nb0mzEWC3IwjPenQ4LSWBN0NNJc2cc68RB+G60xBOEM="; + createNamespace = true; + targetNamespace = "flux-system"; + extraDeploy = [ + { + apiVersion = "fluxcd.controlplane.io/v1"; + kind = "FluxInstance"; + metadata = { + name = "flux"; + namespace = "flux-system"; + annotations = { + "fluxcd.controlplane.io/reconcile" = "enabled"; + "fluxcd.controlplane.io/reconcileEvery" = "1h"; + "fluxcd.controlplane.io/reconcileTimeout" = "5m"; + }; + }; + spec = { + distribution = { + version = "2.x"; + registry = "ghcr.io/fluxcd"; + }; + components = [ + "source-controller" + "kustomize-controller" + "helm-controller" + "notification-controller" + ]; + cluster = { + type = "kubernetes"; + size = "small"; + multitenant = false; + networkPolicy = true; + domain = inputs.secrets.lab.k3s.clusterDomain; + }; + commonMetadata.labels = { + "app.kubernetes.io/name" = "flux"; + }; + sync = ( + { + pullSecret = "git-ssh-key"; + } + // inputs.secrets.lab.k3s.fluxRepo + ); + }; + } + ]; + }; + }; + }; + + # Backup secrets to avoid reissueing them + modules.impermanence.directories = [ + "/opt/k3s-secrets-backup" + ]; + systemd.timers.k3s-secrets-backup-timer = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "15m"; + OnUnitActiveSec = "1h"; + Unit = "k3s-secrets-backup.service"; + }; + }; + systemd.services.k3s-secrets-backup = { + script = '' + mkdir -p /opt/k3s-secrets-backup + touch /opt/k3s-secrets-backup/secrets.yaml + touch /opt/k3s-secrets-backup/namespaces.yaml + chmod 600 /opt/k3s-secrets-backup/secrets.yaml + chmod 600 /opt/k3s-secrets-backup/namespaces.yaml + + ${pkgs.k3s}/bin/kubectl get secrets -A -l controller.cert-manager\.io/fao=="true" -oyaml | ${pkgs.kubectl-neat}/bin/kubectl-neat > /opt/k3s-secrets-backup/secrets.yaml + + echo "apiVersion: v1 + kind: List + items:" > /opt/k3s-secrets-backup/namespaces.yaml + + ${pkgs.gnugrep}/bin/grep -oP '\snamespace: \K.*' /opt/k3s-secrets-backup/secrets.yaml | sort -u | while read -r ns; do + echo "- apiVersion: v1 + kind: Namespace + metadata: + name: $ns" + done >> /opt/k3s-secrets-backup/namespaces.yaml + ''; + serviceConfig = { + Type = "oneshot"; + User = "root"; + }; + }; + + environment.variables = { + KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + }; + + environment.systemPackages = with pkgs; [ + fluxcd + k9s + ]; + + # Use correct disko profile + modules.disko.profile = "k3s"; + + # TEMP: Disable firewall for now + networking.firewall.enable = false; + security.sudo.wheelNeedsPassword = false; +} diff --git a/profiles/disko/k3s.nix b/profiles/disko/k3s.nix new file mode 100644 index 0000000..502fa50 --- /dev/null +++ b/profiles/disko/k3s.nix @@ -0,0 +1,82 @@ +{ + disko.devices = { + disk = { + main = { + type = "disk"; + device = "/dev/sda"; + imageSize = "32G"; # For test VMs + content = { + type = "gpt"; + partitions = { + boot = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + }; + }; + zfs = { + end = "-4G"; + content = { + type = "zfs"; + pool = "tank"; + }; + }; + swap = { + size = "100%"; + content = { + type = "swap"; + discardPolicy = "both"; + }; + }; + }; + }; + }; + longhorn = { + type = "disk"; + device = "/dev/sdb"; + imageSize = "64G"; # For longhorn storage + content = { + type = "gpt"; + partitions = { + main = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + }; + }; + }; + }; + }; + }; + zpool = { + tank = { + type = "zpool"; + rootFsOptions = { + compression = "zstd"; + }; + mountpoint = null; + postCreateHook = "zfs snapshot -r tank@blank && zfs hold -r blank tank@blank"; + + datasets = { + root = { + type = "zfs_fs"; + mountpoint = "/"; + }; + nix = { + type = "zfs_fs"; + mountpoint = "/nix"; + }; + persist = { + type = "zfs_fs"; + mountpoint = "/persist"; + }; + }; + }; + }; + }; +} diff --git a/profiles/nixos/base.nix b/profiles/nixos/base.nix index fff1f78..38931f1 100644 --- a/profiles/nixos/base.nix +++ b/profiles/nixos/base.nix @@ -75,6 +75,7 @@ in zip unzip tmux + unixtools.net-tools ]; }; } diff --git a/profiles/nixos/vm.nix b/profiles/nixos/vm.nix index a2fd20b..100be23 100644 --- a/profiles/nixos/vm.nix +++ b/profiles/nixos/vm.nix @@ -20,7 +20,7 @@ in profiles.base.enable = true; disko = { enable = true; - profile = "vm"; + profile = mkDefault "vm"; }; impermanence = { enable = true;