diff --git a/.forgejo/workflows/nix-flake-check.yaml b/.forgejo/workflows/nix-flake-check.yaml index a4bc8321..19630165 100644 --- a/.forgejo/workflows/nix-flake-check.yaml +++ b/.forgejo/workflows/nix-flake-check.yaml @@ -12,7 +12,7 @@ on: jobs: _checks: - needs: ["deployment-basic","deployment-cli","deployment-model-nixops4","deployment-model-ssh","deployment-model-tf","deployment-panel","nixops-deployment-providers-default","nixops-deployment-providers-fedi200","nixops-deployment-providers-fedi201","nixops-deployment-providers-forgejo-ci","nixops-deployment-providers-test","nixops-deployment-providers-vm02116","nixops-deployment-providers-vm02187","nixosConfigurations-fedi200","nixosConfigurations-fedi201","nixosConfigurations-forgejo-ci","nixosConfigurations-test01","nixosConfigurations-test02","nixosConfigurations-test03","nixosConfigurations-test04","nixosConfigurations-test05","nixosConfigurations-test06","nixosConfigurations-test11","nixosConfigurations-test12","nixosConfigurations-test13","nixosConfigurations-test14","nixosConfigurations-vm02116","nixosConfigurations-vm02187","panel","pre-commit","proxmox-basic","test-mastodon-service","test-peertube-service","vmOptions-fedi200","vmOptions-fedi201","vmOptions-test01","vmOptions-test02","vmOptions-test03","vmOptions-test04","vmOptions-test05","vmOptions-test06","vmOptions-test11","vmOptions-test12","vmOptions-test13","vmOptions-test14"] + needs: ["deployment-basic","deployment-cli","deployment-model-nixops4","deployment-model-ssh","deployment-model-tf","deployment-model-tf-proxmox","deployment-panel","nixops-deployment-providers-default","nixops-deployment-providers-fedi200","nixops-deployment-providers-fedi201","nixops-deployment-providers-forgejo-ci","nixops-deployment-providers-test","nixops-deployment-providers-vm02116","nixops-deployment-providers-vm02187","nixosConfigurations-fedi200","nixosConfigurations-fedi201","nixosConfigurations-forgejo-ci","nixosConfigurations-test01","nixosConfigurations-test02","nixosConfigurations-test03","nixosConfigurations-test04","nixosConfigurations-test05","nixosConfigurations-test06","nixosConfigurations-test11","nixosConfigurations-test12","nixosConfigurations-test13","nixosConfigurations-test14","nixosConfigurations-vm02116","nixosConfigurations-vm02187","panel","pre-commit","proxmox-basic","test-mastodon-service","test-peertube-service","vmOptions-fedi200","vmOptions-fedi201","vmOptions-test01","vmOptions-test02","vmOptions-test03","vmOptions-test04","vmOptions-test05","vmOptions-test06","vmOptions-test11","vmOptions-test12","vmOptions-test13","vmOptions-test14"] runs-on: native steps: - run: true @@ -53,6 +53,12 @@ jobs: - uses: actions/checkout@v4 - run: nix build .#checks.x86_64-linux.deployment-model-tf -vL + deployment-model-tf-proxmox: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix build .#checks.x86_64-linux.deployment-model-tf-proxmox -vL + deployment-panel: runs-on: native steps: diff --git a/deployment/check/basic/default.nix b/deployment/check/basic/default.nix index 176defce..fe53cf4f 100644 --- a/deployment/check/basic/default.nix +++ b/deployment/check/basic/default.nix @@ -1,4 +1,5 @@ { + pkgs, runNixOSTest, inputs, sources, @@ -9,7 +10,10 @@ runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/basic/deployment.nix b/deployment/check/basic/deployment.nix index 14a35ac6..214219f5 100644 --- a/deployment/check/basic/deployment.nix +++ b/deployment/check/basic/deployment.nix @@ -3,6 +3,7 @@ sources, lib, providers, + modulesPath, ... }: @@ -23,7 +24,7 @@ in ../common/targetResource.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { inherit inputs sources modulesPath; }; inherit nodeName pathToRoot pathFromRoot; diff --git a/deployment/check/basic/flake-under-test.nix b/deployment/check/basic/flake-under-test.nix index b9e3fb4b..99542a11 100644 --- a/deployment/check/basic/flake-under-test.nix +++ b/deployment/check/basic/flake-under-test.nix @@ -7,7 +7,12 @@ outputs = inputs: import ./mkFlake.nix inputs ( - { inputs, sources, ... }: + { + inputs, + sources, + modulesPath, + ... + }: { imports = [ inputs.nixops4.modules.flake.default @@ -15,7 +20,7 @@ nixops4Deployments.check-deployment-basic = { imports = [ ./deployment/check/basic/deployment.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { inherit inputs sources modulesPath; }; }; } ); diff --git a/deployment/check/cli/default.nix b/deployment/check/cli/default.nix index 64448280..aed1f301 100644 --- a/deployment/check/cli/default.nix +++ b/deployment/check/cli/default.nix @@ -1,4 +1,5 @@ { + pkgs, runNixOSTest, inputs, sources, @@ -9,7 +10,10 @@ runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/cli/deployments.nix b/deployment/check/cli/deployments.nix index f4e72eaf..b303e291 100644 --- a/deployment/check/cli/deployments.nix +++ b/deployment/check/cli/deployments.nix @@ -2,6 +2,7 @@ inputs, sources, lib, + modulesPath, }: let @@ -15,7 +16,7 @@ let makeTargetResource = nodeName: { imports = [ ../common/targetResource.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { inherit inputs sources modulesPath; }; inherit nodeName pathToRoot diff --git a/deployment/check/cli/flake-under-test.nix b/deployment/check/cli/flake-under-test.nix index fa172890..edfd3685 100644 --- a/deployment/check/cli/flake-under-test.nix +++ b/deployment/check/cli/flake-under-test.nix @@ -11,6 +11,7 @@ inputs, sources, lib, + modulesPath, ... }: { @@ -19,7 +20,12 @@ ]; nixops4Deployments = import ./deployment/check/cli/deployments.nix { - inherit inputs sources lib; + inherit + inputs + sources + lib + modulesPath + ; }; } ); diff --git a/deployment/check/common/deployerNode.nix b/deployment/check/common/deployerNode.nix index dcb5deef..1948e388 100644 --- a/deployment/check/common/deployerNode.nix +++ b/deployment/check/common/deployerNode.nix @@ -3,6 +3,7 @@ lib, pkgs, config, + modulesPath, sources, ... }: @@ -75,6 +76,7 @@ in machine = (pkgs.nixos [ ./targetNode.nix + "${modulesPath}/../lib/testing/nixos-test-base.nix" config.system.extraDependenciesFromModule { nixpkgs.hostPlatform = "x86_64-linux"; diff --git a/deployment/check/common/nixosTest.nix b/deployment/check/common/nixosTest.nix index 9d4e528d..ac2c6b4d 100644 --- a/deployment/check/common/nixosTest.nix +++ b/deployment/check/common/nixosTest.nix @@ -4,6 +4,7 @@ config, hostPkgs, sources, + modulesPath, ... }: @@ -112,7 +113,9 @@ in }; }) (genAttrs config.targetMachines (_: { - imports = [ ./targetNode.nix ]; + imports = [ + ./targetNode.nix + ] ++ (lib.optional config.useFlake "${modulesPath}/../lib/testing/nixos-test-base.nix"); _module.args = { inherit inputs sources; }; enableAcme = config.enableAcme; acmeNodeIP = if config.enableAcme then config.nodes.acme.networking.primaryIPAddress else null; diff --git a/deployment/check/common/targetNode.nix b/deployment/check/common/targetNode.nix index 2592b5fe..682a247a 100644 --- a/deployment/check/common/targetNode.nix +++ b/deployment/check/common/targetNode.nix @@ -15,9 +15,8 @@ in _class = "nixos"; imports = [ - (modulesPath + "/profiles/minimal.nix") - (modulesPath + "/profiles/qemu-guest.nix") - (modulesPath + "/../lib/testing/nixos-test-base.nix") + "${modulesPath}/profiles/minimal.nix" + "${modulesPath}/profiles/qemu-guest.nix" ./sharedOptions.nix ]; @@ -43,8 +42,8 @@ in networking.firewall.allowedTCPPorts = [ 22 ]; - ## Test VMs don't have a bootloader by default. - boot.loader.grub.enable = false; + # Test VMs don't have a bootloader by default. + boot.loader.grub.enable = lib.mkDefault false; } (mkIf config.enableAcme { diff --git a/deployment/check/common/targetResource.nix b/deployment/check/common/targetResource.nix index 53e07f16..3e7eb2e1 100644 --- a/deployment/check/common/targetResource.nix +++ b/deployment/check/common/targetResource.nix @@ -3,6 +3,7 @@ lib, config, sources, + modulesPath, ... }: @@ -39,6 +40,7 @@ in imports = [ ./targetNode.nix (lib.modules.importJSON (config.pathToCwd + "/${config.nodeName}-network.json")) + "${modulesPath}/../lib/testing/nixos-test-base.nix" ]; _module.args = { inherit inputs sources; }; diff --git a/deployment/check/common/utils.nix b/deployment/check/common/utils.nix index ba27c050..14001cda 100644 --- a/deployment/check/common/utils.nix +++ b/deployment/check/common/utils.nix @@ -1,6 +1,6 @@ { lib, - sources ? import ../../../npins, + modulesPath, ... }: { @@ -11,7 +11,7 @@ imports = [ ../common/sharedOptions.nix ../common/targetNode.nix - "${sources.nixpkgs}/nixos/modules/profiles/qemu-guest.nix" + "${modulesPath}/profiles/qemu-guest.nix" ]; users.users = environment.config.resources."operator-environment".login-shell.apply { diff --git a/deployment/check/data-model-nixops4/data-model.nix b/deployment/check/data-model-nixops4/data-model.nix index 7ddbe3d9..a1b9a3d3 100644 --- a/deployment/check/data-model-nixops4/data-model.nix +++ b/deployment/check/data-model-nixops4/data-model.nix @@ -1,6 +1,7 @@ { config, system, + modulesPath, inputs, sources ? import ../../../npins, ... @@ -9,7 +10,7 @@ let inherit (sources) nixpkgs; pkgs = import nixpkgs { inherit system; }; inherit (pkgs) lib; - inherit (pkgs.callPackage ../common/utils.nix { }) mkNixosConfiguration; + inherit (pkgs.callPackage ../common/utils.nix { inherit modulesPath; }) mkNixosConfiguration; inherit (config) nodeName pathFromRoot @@ -17,7 +18,7 @@ let ; in (pkgs.callPackage ../../utils.nix { inherit inputs; }).evalModel ( - { config, ... }: + { config, modulesPath, ... }: { imports = [ ../common/model.nix ]; config = { @@ -41,8 +42,13 @@ in inputs.nixops4-nixos.modules.nixops4Resource.nixos ../common/targetResource.nix ]; - nixos.module = mkNixosConfiguration environment required-resources; - _module.args = { inherit inputs sources; }; + nixos.module = { + imports = [ + (mkNixosConfiguration environment required-resources) + "${modulesPath}/../lib/testing/nixos-test-base.nix" + ]; + }; + _module.args = { inherit inputs sources modulesPath; }; inherit nodeName pathToRoot pathFromRoot; }; }; diff --git a/deployment/check/data-model-nixops4/default.nix b/deployment/check/data-model-nixops4/default.nix index a08d4cd4..4041c0a0 100644 --- a/deployment/check/data-model-nixops4/default.nix +++ b/deployment/check/data-model-nixops4/default.nix @@ -1,4 +1,5 @@ { + pkgs, runNixOSTest, inputs, sources, @@ -11,7 +12,10 @@ runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/data-model-nixops4/flake-under-test.nix b/deployment/check/data-model-nixops4/flake-under-test.nix index e9dbc484..38bbd0a9 100644 --- a/deployment/check/data-model-nixops4/flake-under-test.nix +++ b/deployment/check/data-model-nixops4/flake-under-test.nix @@ -7,7 +7,7 @@ outputs = inputs: import ./mkFlake.nix inputs ( - { inputs, ... }: + { inputs, modulesPath, ... }: let system = "x86_64-linux"; in @@ -18,7 +18,7 @@ nixops4Deployments.check-deployment-model = (import ./deployment/check/data-model-nixops4/data-model.nix { - inherit system inputs; + inherit system inputs modulesPath; config = { inherit (import ./deployment/check/data-model-nixops4/constants.nix) pathToRoot pathFromRoot; nodeName = "nixops4"; diff --git a/deployment/check/data-model-ssh/data-model.nix b/deployment/check/data-model-ssh/data-model.nix index d3df53bf..91aa69b1 100644 --- a/deployment/check/data-model-ssh/data-model.nix +++ b/deployment/check/data-model-ssh/data-model.nix @@ -1,6 +1,7 @@ { config, system, + modulesPath, sources ? import ../../../npins, ... }@args: @@ -8,7 +9,7 @@ let inherit (sources) nixpkgs; pkgs = import nixpkgs { inherit system; }; inherit (pkgs) lib; - inherit (pkgs.callPackage ../common/utils.nix { }) mkNixosConfiguration; + inherit (pkgs.callPackage ../common/utils.nix { inherit modulesPath; }) mkNixosConfiguration; inherit (config) nodeName pathToRoot @@ -17,7 +18,7 @@ let ; in (pkgs.callPackage ../../utils.nix { }).evalModel ( - { config, ... }: + { config, modulesPath, ... }: { imports = [ ../common/model.nix ]; config = { @@ -31,7 +32,12 @@ in }: { ssh-host = { - nixos-configuration = mkNixosConfiguration environment required-resources; + nixos-configuration = { + imports = [ + (mkNixosConfiguration environment required-resources) + "${modulesPath}/../lib/testing/nixos-test-base.nix" + ]; + }; system = targetSystem; ssh = { username = "root"; diff --git a/deployment/check/data-model-ssh/default.nix b/deployment/check/data-model-ssh/default.nix index 86ddafb9..5adb5c89 100644 --- a/deployment/check/data-model-ssh/default.nix +++ b/deployment/check/data-model-ssh/default.nix @@ -1,4 +1,5 @@ { + pkgs, runNixOSTest, inputs, sources, @@ -11,7 +12,10 @@ runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/data-model-ssh/nixosTest.nix b/deployment/check/data-model-ssh/nixosTest.nix index 5efdadc8..f97ccaa1 100644 --- a/deployment/check/data-model-ssh/nixosTest.nix +++ b/deployment/check/data-model-ssh/nixosTest.nix @@ -1,6 +1,7 @@ { lib, pkgs, + modulesPath, ... }: let @@ -8,7 +9,7 @@ let nodeName = "ssh"; deploy = (import ./data-model.nix { - inherit system; + inherit system modulesPath; config = { inherit nodeName; inherit (import ./constants.nix) pathToRoot; diff --git a/deployment/check/data-model-tf-proxmox/constants.nix b/deployment/check/data-model-tf-proxmox/constants.nix new file mode 100644 index 00000000..4c96195d --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/constants.nix @@ -0,0 +1,10 @@ +{ + targetMachines = [ + "pve" + ]; + pathToRoot = builtins.path { + path = ../../..; + name = "root"; + }; + pathFromRoot = "/deployment/check/data-model-tf-proxmox"; +} diff --git a/deployment/check/data-model-tf-proxmox/default.nix b/deployment/check/data-model-tf-proxmox/default.nix new file mode 100644 index 00000000..16fedf49 --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/default.nix @@ -0,0 +1,54 @@ +{ + inputs, + sources, + system, +}: + +let + pkgs = import sources.nixpkgs-stable { + inherit system; + overlays = [ overlay ]; + }; + overlay = _: prev: { + terraform-backend = + prev.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/te/terraform-backend/package.nix" + { }; + inherit + (import "${sources.proxmox-nixos}/pkgs" { + craneLib = pkgs.callPackage "${sources.crane}/lib" { }; + # breaks from https://github.com/NixOS/nixpkgs/commit/06b354eb2dc535c57e9b4caaa16d79168f117a26, + # which updates libvncserver to 0.9.15, which was not yet patched at https://git.proxmox.com/?p=vncterm.git. + inherit pkgs; + # not so picky about version for our purposes + pkgs-unstable = pkgs; + }) + proxmox-ve + pve-manager + pve-ha-manager + pve-qemu + ; + }; +in +pkgs.testers.runNixOSTest { + node.specialArgs = { + inherit + sources + pkgs + ; + }; + imports = [ + ../../data-model.nix + ../../function.nix + ../common/nixosTest.nix + ./nixosTest.nix + ]; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; + inherit (import ./constants.nix) + targetMachines + pathToRoot + pathFromRoot + ; +} diff --git a/deployment/check/data-model-tf-proxmox/nixosTest.nix b/deployment/check/data-model-tf-proxmox/nixosTest.nix new file mode 100644 index 00000000..d2c6dc29 --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/nixosTest.nix @@ -0,0 +1,256 @@ +{ + lib, + pkgs, + modulesPath, + sources, + ... +}: +let + inherit (pkgs) system; + backendPort = builtins.toString 8080; + tfBackend = fragment: rec { + TF_HTTP_USERNAME = "basic"; + TF_HTTP_PASSWORD = "fake-secret"; + TF_HTTP_LOCK_ADDRESS = TF_HTTP_ADDRESS; + TF_HTTP_UNLOCK_ADDRESS = TF_HTTP_ADDRESS; + TF_HTTP_ADDRESS = "http://localhost:${backendPort}/state/${fragment}"; + }; + template-deployment = + (import ./setups/template.nix { + inherit sources system modulesPath; + config = { + httpBackend = tfBackend "proxmox-test/upload"; + nodeName = "pve"; + targetSystem = system; + node-name = "pve"; + imageDatastoreId = "local"; + }; + }).default.tf-proxmox-template; + vm-deployment = + (import ./setups/vm.nix { + inherit sources system modulesPath; + config = { + httpBackend = tfBackend "proxmox-test/nixos"; + inherit (import ./constants.nix) pathToRoot; + nodeName = "pve"; + targetSystem = system; + # for the test use the proxmox host as jump host, + # as we have no static IPs the deployer can reach the deployed VM on + sshOpts = [ + "ProxyCommand=ssh -W %h:%p pve" + ]; + key-file = "/root/.ssh/id_ed25519"; + node-name = "pve"; + bridge = "br0"; + vlanId = 0; + imageDatastoreId = "local"; + vmDatastoreId = "local"; + cdDatastoreId = "local"; + ipv4Gateway = "192.168.10.1"; + ipv4Address = "192.168.10.236/24"; + ipv6Gateway = ""; + ipv6Address = ""; + # dynamically get the id from the template upload step + templateId = null; + }; + }).default.tf-proxmox-vm; +in +{ + _class = "nixosTest"; + name = "deployment-model"; + sourceFileset = lib.fileset.unions [ + ../../run/tf-proxmox-template/run.sh + ../../run/tf-proxmox-vm/run.sh + ../../run/tf-proxmox-vm/await-ssh.sh + ]; + + nodes.pve = + { sources, ... }: + { + imports = [ + "${sources.proxmox-nixos}/modules/proxmox-ve" + ]; + environment.systemPackages = [ + pkgs.jq + pkgs.qemu + ]; + networking.firewall.enable = false; + networking.vlans = { + vlan0 = { + id = 0; + interface = "eth0"; + }; + }; + networking.useDHCP = false; + + networking = { + bridges.br0.interfaces = [ ]; + interfaces.br0.ipv4.addresses = [ + { + address = "192.168.10.1"; + prefixLength = 24; + } + ]; + nat = { + enable = true; + internalInterfaces = [ "br0" ]; + }; + }; + boot.kernel.sysctl."net.ipv4.ip_forward" = "1"; + + users.users.root = { + password = "mytestpw"; + hashedPasswordFile = lib.mkForce null; + }; + # https://github.com/SaumonNet/proxmox-nixos/blob/main/modules/proxmox-ve/default.nix + services.proxmox-ve = { + enable = true; + ipAddress = "192.168.1.1"; + }; + virtualisation = { + diskSize = 5 * 1024; + memorySize = 3 * 1024; + }; + }; + + nodes.deployer = + { ... }: + { + imports = [ + ../../modules/terraform-backend + ]; + + networking.firewall.enable = false; + nix.nixPath = [ + (lib.concatStringsSep ":" (lib.mapAttrsToList (k: v: k + "=" + v) sources)) + ]; + + environment.systemPackages = [ + vm-deployment.run + template-deployment.run + pkgs.pve-manager + pkgs.openssl + pkgs.jq + (pkgs.callPackage ../../run/tf-proxmox-template/tf.nix { }) + (pkgs.callPackage ../../run/tf-proxmox-vm/tf.nix { }) + ]; + + # needed only when building from deployer + system.extraDependenciesFromModule = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + hello + ]; + }; + system.extraDependencies = [ + sources.disko + pkgs.ubootQemuX86 + pkgs.ubootQemuX86.inputDerivation + pkgs.pve-qemu + pkgs.pve-qemu.inputDerivation + pkgs.gnu-config + pkgs.byacc + pkgs.stdenv + pkgs.stdenvNoCC + sources.nixpkgs + pkgs.vte + ]; + services.terraform-backend = { + enable = true; + settings = { + LISTEN_ADDR = ":${backendPort}"; + # FIXME randomly generate this + KMS_KEY = "tsjxw9NjKUBUlzbTnD7orqIAdEmpGYRARvxD51jtY+o="; + }; + }; + }; + + extraTestScript = '' + pve.wait_for_unit("pveproxy.service") + assert "running" in pve.succeed("pveproxy status") + pve.succeed("mkdir -p /run/pve") + assert "Proxmox" in pve.succeed("curl -s -i -k https://localhost:8006") + + cert = pve.succeed("cat /etc/pve/pve-root-ca.pem").strip() + + pve.succeed("pvesh create /pools --poolid Fediversity") + + # allow upload of `import` (template) files + pve.succeed(""" + pvesh set /storage/local --content "vztmpl,rootdir,backup,snippets,import,iso,images" 1>/dev/null + """) + + template_token = pve.succeed(""" + pvesh create /access/users/root@pam/token/template --output-format json | jq -r .value + pvesh set /access/acl --path "/" --token "root@pam!template" --roles "PVEDatastoreAdmin" + """).strip() + + vm_token = pve.succeed(""" + pvesh create /access/users/root@pam/token/vm --output-format json | jq -r .value + pvesh set /access/acl --path "/" --token "root@pam!vm" --roles "PVEVMAdmin PVEDatastoreAdmin PVESDNUser" + """).strip() + + # skip indent for EOF + deployer.succeed(f""" + cat > /etc/ssl/certs/pve-root-ca.pem < /root/.ssh/id_ed25519 < new-ca-bundle.crt + rm ca-bundle.crt ca-certificates.crt + mv new-ca-bundle.crt ca-bundle.crt + ln -s ca-bundle.crt ca-certificates.crt + openssl verify -CApath /etc/ssl/certs ./pve-root-ca.pem + """) + + with subtest("Deploy the template"): + template_id = deployer.succeed(f""" + ssh -o BatchMode=yes -o StrictHostKeyChecking=no pve "true" + export PROXMOX_VE_INSECURE="true" + export SSL_CERT_FILE=/tmp/pve-ca-bundle.crt + export PROXMOX_VE_API_TOKEN="root@pam!template={template_token}" + ${lib.getExe template-deployment.run} | jq -r '.id.value' + """).strip() + + deploy = f""" + set -e + ssh -o BatchMode=yes -o StrictHostKeyChecking=no pve "true" + export PROXMOX_VE_INSECURE="true" + export SSL_CERT_FILE=/tmp/pve-ca-bundle.crt + export PROXMOX_VE_API_TOKEN="root@pam!vm={vm_token}" + export TF_VAR_template_id="{template_id}" + ${lib.getExe vm-deployment.run} | jq -r '.ipv4.value[0]' + """ + + with subtest("Run the deployment"): + ip = deployer.succeed(deploy).strip() + + with subtest("Verify package"): + deployer.succeed(f""" + ssh -i "/root/.ssh/id_ed25519" -o StrictHostKeyChecking=no -o BatchMode=yes -J pve root@{ip} su - operator -c hello >&2 + """) + + with subtest("No-op update"): + deployer.succeed(deploy, timeout=120) + ''; +} diff --git a/deployment/check/data-model-tf-proxmox/setups/shared.nix b/deployment/check/data-model-tf-proxmox/setups/shared.nix new file mode 100644 index 00000000..3ea6b94e --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/setups/shared.nix @@ -0,0 +1,67 @@ +{ + lib, + sources ? import ../../../../npins, + ... +}: +{ + imports = [ + "${sources.disko}/module.nix" + ../../../../infra/common/proxmox-qemu-vm.nix + ]; + networking.useDHCP = false; + services.qemuGuest.enable = true; + services.openssh.enable = true; + services.cloud-init = { + enable = true; + network.enable = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFZsldWMEsajYysjYsEpNvMOjO4D8L21pTrfQS1T+Hfy" + ]; + boot.loader = { + systemd-boot.enable = true; + efi = { + canTouchEfiVariables = true; + efiSysMountPoint = "/boot"; + }; + grub.enable = false; + }; + fileSystems."/boot" = { + fsType = "vfat"; + device = lib.mkDefault "/dev/sda1"; + options = [ + "fmask=0022" + "dmask=0022" + ]; + }; + disko.devices.disk.main = { + device = "/dev/sda"; + type = "disk"; + imageSize = "20G"; # needed for image generation + content = { + type = "gpt"; + partitions = { + esp = { + priority = 1; + size = "500M"; + type = "EF00"; + label = "boot"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + priority = 2; + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; +} diff --git a/deployment/check/data-model-tf-proxmox/setups/template.nix b/deployment/check/data-model-tf-proxmox/setups/template.nix new file mode 100644 index 00000000..983ef8c9 --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/setups/template.nix @@ -0,0 +1,70 @@ +{ + config, + system, + modulesPath, + sources ? import ../../../../npins, + ... +}: +let + inherit (sources) nixpkgs; + pkgs = import nixpkgs { inherit system; }; + inherit (pkgs) lib; + inherit (pkgs.callPackage ../../common/utils.nix { inherit modulesPath; }) mkNixosConfiguration; + inherit (config) + nodeName + targetSystem + httpBackend + node-name + imageDatastoreId + ; +in +(pkgs.callPackage ../../../utils.nix { }).evalModel ( + { config, ... }: + { + imports = [ ../../common/model.nix ]; + config = { + environments.default = environment: { + resources."operator-environment".login-shell = { + wheel = true; + username = "operator"; + }; + implementation = + { + required-resources, + ... + }: + { + tf-proxmox-template = { + nixos-configuration = { + imports = [ + (mkNixosConfiguration environment required-resources) + ./shared.nix + ]; + }; + system = targetSystem; + ssh = { + host = nodeName; + }; + inherit + node-name + httpBackend + imageDatastoreId + ; + }; + }; + }; + }; + options.default = + let + env = config.environments.default; + in + lib.mkOption { + type = env.resource-mapping.output-type; + default = env.deployment { + deployment-name = "default"; + # normally our template is distinct, but our test cannot download build deps due to sandboxing + configuration = config."example-configuration"; + }; + }; + } +) diff --git a/deployment/check/data-model-tf-proxmox/setups/vm.nix b/deployment/check/data-model-tf-proxmox/setups/vm.nix new file mode 100644 index 00000000..167eb8e3 --- /dev/null +++ b/deployment/check/data-model-tf-proxmox/setups/vm.nix @@ -0,0 +1,96 @@ +{ + config, + system, + modulesPath, + sources ? import ../../../../npins, + ... +}@args: +let + inherit (sources) nixpkgs; + pkgs = import nixpkgs { inherit system; }; + inherit (pkgs) lib; + inherit (pkgs.callPackage ../../common/utils.nix { inherit modulesPath; }) mkNixosConfiguration; + inherit (config) + nodeName + pathToRoot + targetSystem + sshOpts + httpBackend + key-file + node-name + bridge + vlanId + templateId + imageDatastoreId + vmDatastoreId + cdDatastoreId + ipv4Gateway + ipv4Address + ipv6Gateway + ipv6Address + ; +in +(pkgs.callPackage ../../../utils.nix { }).evalModel ( + { config, ... }: + { + imports = [ ../../common/model.nix ]; + config = { + environments.default = environment: { + resources."operator-environment".login-shell = { + wheel = true; + username = "operator"; + }; + implementation = + { + required-resources, + deployment-name, + }: + { + tf-proxmox-vm = { + nixos-configuration = { + imports = [ + (mkNixosConfiguration environment required-resources) + ./shared.nix + ]; + }; + system = targetSystem; + ssh = { + username = "root"; + host = nodeName; + inherit key-file sshOpts; + }; + caller = "deployment/check/data-model-tf-proxmox/setups/vm.nix"; + inherit + args + deployment-name + httpBackend + node-name + bridge + vlanId + templateId + imageDatastoreId + vmDatastoreId + cdDatastoreId + ipv4Gateway + ipv4Address + ipv6Gateway + ipv6Address + ; + root-path = pathToRoot; + }; + }; + }; + }; + options.default = + let + env = config.environments.default; + in + lib.mkOption { + type = env.resource-mapping.output-type; + default = env.deployment { + deployment-name = "default"; + configuration = config."example-configuration"; + }; + }; + } +) diff --git a/deployment/check/data-model-tf/data-model.nix b/deployment/check/data-model-tf/data-model.nix index a765fdc1..e204475f 100644 --- a/deployment/check/data-model-tf/data-model.nix +++ b/deployment/check/data-model-tf/data-model.nix @@ -2,13 +2,14 @@ config, system, sources ? import ../../../npins, + modulesPath, ... }@args: let inherit (sources) nixpkgs; pkgs = import nixpkgs { inherit system; }; inherit (pkgs) lib; - inherit (pkgs.callPackage ../common/utils.nix { }) mkNixosConfiguration; + inherit (pkgs.callPackage ../common/utils.nix { inherit modulesPath; }) mkNixosConfiguration; inherit (config) nodeName pathToRoot @@ -18,7 +19,7 @@ let ; in (pkgs.callPackage ../../utils.nix { }).evalModel ( - { config, ... }: + { config, modulesPath, ... }: { imports = [ ../common/model.nix ]; config = { @@ -31,7 +32,12 @@ in }: { tf-host = { - nixos-configuration = mkNixosConfiguration environment required-resources; + nixos-configuration = { + imports = [ + (mkNixosConfiguration environment required-resources) + "${modulesPath}/../lib/testing/nixos-test-base.nix" + ]; + }; system = targetSystem; ssh = { username = "root"; diff --git a/deployment/check/data-model-tf/default.nix b/deployment/check/data-model-tf/default.nix index 8361293e..9fb576a6 100644 --- a/deployment/check/data-model-tf/default.nix +++ b/deployment/check/data-model-tf/default.nix @@ -43,7 +43,10 @@ pkgs.testers.runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/data-model-tf/nixosTest.nix b/deployment/check/data-model-tf/nixosTest.nix index 4bbca5f1..46ea4d35 100644 --- a/deployment/check/data-model-tf/nixosTest.nix +++ b/deployment/check/data-model-tf/nixosTest.nix @@ -2,6 +2,7 @@ lib, pkgs, sources, + modulesPath, ... }: let @@ -11,7 +12,7 @@ let backendPort = builtins.toString 8080; deploy = (import ./data-model.nix { - inherit system; + inherit system modulesPath; config = { inherit nodeName pathToRoot; targetSystem = system; diff --git a/deployment/check/panel/default.nix b/deployment/check/panel/default.nix index 64448280..aed1f301 100644 --- a/deployment/check/panel/default.nix +++ b/deployment/check/panel/default.nix @@ -1,4 +1,5 @@ { + pkgs, runNixOSTest, inputs, sources, @@ -9,7 +10,10 @@ runNixOSTest { ../common/nixosTest.nix ./nixosTest.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; inherit (import ./constants.nix) targetMachines pathToRoot diff --git a/deployment/check/panel/deployment.nix b/deployment/check/panel/deployment.nix index cb3618dd..82374134 100644 --- a/deployment/check/panel/deployment.nix +++ b/deployment/check/panel/deployment.nix @@ -2,6 +2,7 @@ inputs, sources, lib, + modulesPath, }: let @@ -15,7 +16,7 @@ let makeTargetResource = nodeName: { imports = [ ../common/targetResource.nix ]; - _module.args = { inherit inputs sources; }; + _module.args = { inherit inputs sources modulesPath; }; inherit nodeName pathToRoot diff --git a/deployment/check/panel/flake-under-test.nix b/deployment/check/panel/flake-under-test.nix index 23ecec02..b526a670 100644 --- a/deployment/check/panel/flake-under-test.nix +++ b/deployment/check/panel/flake-under-test.nix @@ -11,6 +11,7 @@ inputs, sources, lib, + modulesPath, ... }: { @@ -19,7 +20,12 @@ ]; nixops4Deployments.check-deployment-panel = import ./deployment/check/panel/deployment.nix { - inherit inputs sources lib; + inherit + inputs + sources + lib + modulesPath + ; }; } ); diff --git a/deployment/check/proxmox/default.nix b/deployment/check/proxmox/default.nix index 78aa4046..84f35fbd 100644 --- a/deployment/check/proxmox/default.nix +++ b/deployment/check/proxmox/default.nix @@ -30,6 +30,7 @@ runNixOSTest { sources pkgs ; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; }; imports = [ ./proxmoxTest.nix diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 93295d51..863c25b4 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -3,6 +3,7 @@ lib, config, inputs, + sources ? import ../npins, ... }: let @@ -283,6 +284,284 @@ let }; }); }; + tf-proxmox-template = mkOption { + description = '' + A Terraform deployment to upload a virtual machine template to ProxmoX VE. + Proxmox credentials should be set using [environment variables] + (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) + with role `PVEDatastoreAdmin`. + ''; + type = submodule (tf-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit nixos-configuration; + ssh = host-ssh; + node-name = mkOption { + description = "the name of the ProxmoX node to use."; + type = types.str; + }; + httpBackend = mkOption { + description = "environment variables to configure the TF HTTP back-end, see "; + type = types.attrsOf (types.either types.str types.int); + }; + imageDatastoreId = mkOption { + description = "ID of the datastore of the image."; + type = types.str; + default = "local"; + }; + run = mkOption { + type = types.package; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + httpBackend + node-name + imageDatastoreId + ; + inherit (ssh) + host + ; + machine = import ./nixos.nix { + inherit sources system; + configuration = tf-host.config.nixos-configuration; + }; + name = "fediversity-template"; + + # worse for cross-compilation, better for pre-/post-processing, needs manual `imageSize`, random failures: https://github.com/nix-community/disko/issues/550#issuecomment-2503736973 + raw = "${machine.config.system.build.diskoImages}/main.raw"; + + environment = { + inherit + host + ; + node_name = node-name; + image_datastore_id = imageDatastoreId; + }; + tf-env = pkgs.callPackage ./run/tf-env.nix { + inherit httpBackend; + tfPackage = pkgs.callPackage ./run/tf-proxmox-template/tf.nix { }; + tfDirs = [ + "deployment/run/tf-proxmox-template" + ]; + }; + in + lib.trace (lib.strings.toJSON environment) pkgs.writers.writeBashBin "deploy-tf-proxmox-template.sh" + (withPackages [ + pkgs.jq + pkgs.qemu + (pkgs.callPackage ./run/tf-proxmox-vm/tf.nix { }) + ]) + '' + set -e + + # nixos-generate gives the burden of building revisions, while systemd-repart handles partitioning ~~at the burden of version revisions~~ + # .qcow2 is around half the size of .raw, on top of supporting backups - be it apparently at the cost of performance + qemu-img convert -f raw -O qcow2 -C "${raw}" /tmp/${name}.qcow2 + + ls -l ${raw} >&2 + ls -l /tmp/${name}.qcow2 >&2 + checksum="$(sha256sum /tmp/${name}.qcow2 | cut -d " " -f1)" + + env ${toString (lib.mapAttrsToList (k: v: "TF_VAR_${k}=\"${toBash v}\"") environment)} \ + ${toString (lib.mapAttrsToList (k: v: "${k}=\"${toBash v}\"") httpBackend)} \ + TF_VAR_image=/tmp/${name}.qcow2 \ + TF_VAR_checksum="$checksum" \ + tf_env=${tf-env} bash ./deployment/run/tf-proxmox-template/run.sh + ''; + }; + }; + }); + }; + tf-proxmox-vm = mkOption { + description = '' + A Terraform deployment to provision and update a virtual machine on ProxmoX VE. + Proxmox credentials should be set using [environment variables] + (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) + with roles `PVEVMAdmin PVEDatastoreAdmin PVESDNUser`. + ''; + type = submodule (tf-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit nixos-configuration; + ssh = host-ssh; + caller = mkOption { + description = "The calling module to obtain the NixOS configuration from."; + type = types.str; + }; + args = mkOption { + description = "The arguments with which to call the module to obtain the NixOS configuration."; + type = types.attrs; + }; + deployment-name = mkOption { + description = "The name of the deployment for which to obtain the NixOS configuration."; + type = types.str; + }; + root-path = mkOption { + description = "The path to the root of the repository."; + type = types.path; + }; + node-name = mkOption { + description = "the name of the ProxmoX node to use."; + type = types.str; + }; + httpBackend = mkOption { + description = "environment variables to configure the TF HTTP back-end, see "; + type = types.attrsOf (types.either types.str types.int); + }; + bridge = mkOption { + description = "The name of the network bridge (defaults to vmbr0)."; + type = types.str; + default = "vmbr0"; + }; + vlanId = mkOption { + description = "The VLAN identifier."; + type = types.int; + default = 0; + }; + imageDatastoreId = mkOption { + description = "ID of the datastore of the image."; + type = types.str; + default = "local"; + }; + templateId = mkOption { + description = "ID of the template file from which to clone the VM."; + type = types.nullOr types.str; + example = "local:import/template.qcow2"; + }; + vmDatastoreId = mkOption { + description = "ID of the datastore of the VM."; + type = types.str; + default = "local"; + }; + cdDatastoreId = mkOption { + description = "ID of the datastore of the virtual CD-rom drive to use for cloud-init."; + type = types.str; + default = "local"; + }; + ipv4Gateway = mkOption { + description = "Gateway for IPv4."; + type = types.str; + default = ""; + }; + ipv4Address = mkOption { + description = "IPv4 address."; + type = types.str; + default = ""; + }; + ipv6Gateway = mkOption { + description = "Gateway for IPv6."; + type = types.str; + default = ""; + }; + ipv6Address = mkOption { + description = "IPv6 address."; + type = types.str; + default = ""; + }; + run = mkOption { + type = types.package; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + caller + args + deployment-name + httpBackend + root-path + node-name + bridge + vlanId + imageDatastoreId + templateId + vmDatastoreId + cdDatastoreId + ipv4Gateway + ipv4Address + ipv6Gateway + ipv6Address + ; + inherit (ssh) + host + username + key-file + sshOpts + ; + deployment-type = "tf-proxmox-vm"; + nixos_conf = writeConfig { + inherit + system + caller + args + deployment-name + root-path + deployment-type + ; + }; + environment = { + key_file = key-file; + ssh_opts = sshOpts; + inherit + host + nixos_conf + bridge + ; + node_name = node-name; + ssh_user = username; + vlan_id = vlanId; + image_datastore_id = imageDatastoreId; + template_id = templateId; + vm_datastore_id = vmDatastoreId; + cd_datastore_id = cdDatastoreId; + ipv4_gateway = ipv4Gateway; + ipv4_address = ipv4Address; + ipv6_gateway = ipv6Gateway; + ipv6_address = ipv6Address; + }; + tf-env = pkgs.callPackage ./run/tf-env.nix { + inherit httpBackend; + tfPackage = pkgs.callPackage ./run/tf-proxmox-vm/tf.nix { }; + tfDirs = [ + "deployment/run/tf-single-host" + "deployment/run/tf-proxmox-vm" + ]; + }; + in + lib.trace (lib.strings.toJSON environment) pkgs.writers.writeBashBin "deploy-tf-proxmox-vm.sh" + (withPackages [ + pkgs.jq + pkgs.qemu + (pkgs.callPackage ./run/tf-proxmox-vm/tf.nix { }) + ]) + '' + set -e + env ${ + toString ( + lib.mapAttrsToList (k: v: "TF_VAR_${k}=\"${toBash v}\"") ( + lib.filterAttrs (_: v: v != null) environment + ) + ) + } \ + ${toString (lib.mapAttrsToList (k: v: "${k}=\"${toBash v}\"") httpBackend)} \ + tf_env=${tf-env} bash ./deployment/run/tf-proxmox-vm/run.sh + ''; + }; + }; + }); + }; }; in { diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 5d058499..002e71c8 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -14,32 +14,36 @@ deployment-basic = import ./check/basic { inherit (pkgs.testers) runNixOSTest; - inherit inputs sources; + inherit pkgs inputs sources; }; deployment-cli = import ./check/cli { inherit (pkgs.testers) runNixOSTest; - inherit inputs sources; + inherit pkgs inputs sources; }; deployment-panel = import ./check/panel { inherit (pkgs.testers) runNixOSTest; - inherit inputs sources; + inherit pkgs inputs sources; }; deployment-model-ssh = import ./check/data-model-ssh { inherit (pkgs.testers) runNixOSTest; - inherit inputs sources; + inherit pkgs inputs sources; }; deployment-model-nixops4 = import ./check/data-model-nixops4 { inherit (pkgs.testers) runNixOSTest; - inherit inputs sources; + inherit pkgs inputs sources; }; deployment-model-tf = import ./check/data-model-tf { inherit inputs sources system; }; + + deployment-model-tf-proxmox = import ./check/data-model-tf-proxmox { + inherit inputs sources system; + }; }; }; } diff --git a/deployment/nixos.nix b/deployment/nixos.nix index 5dde156e..aba1202f 100644 --- a/deployment/nixos.nix +++ b/deployment/nixos.nix @@ -1,9 +1,10 @@ { configuration, system, - sources ? import ../npins, + ... }: let + sources = import ../npins; eval = import "${sources.nixpkgs}/nixos/lib/eval-config.nix" { inherit system; specialArgs = { diff --git a/deployment/run/ssh-single-host/run.sh b/deployment/run/ssh-single-host/run.sh index 84c3f7cb..b3766d1c 100755 --- a/deployment/run/ssh-single-host/run.sh +++ b/deployment/run/ssh-single-host/run.sh @@ -47,4 +47,4 @@ NIX_SSHOPTS="${sshOptsAsterisk[*]}" nix-copy-closure --to "$destination" "$outPa # shellcheck disable=SC2029 ssh "${sshOptsAt[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath" # shellcheck disable=SC2029 -ssh -o "ConnectTimeout=5" -o "ServerAliveInterval=1" "${sshOptsAt[@]}" "$destination" "nohup env $outPath/bin/switch-to-configuration switch &" 2>&1 +ssh -o "ConnectTimeout=5" -o "ServerAliveInterval=1" "${sshOptsAt[@]}" "$destination" "nohup env NIXOS_INSTALL_BOOTLOADER=0 $outPath/bin/switch-to-configuration switch &" 2>&1 diff --git a/deployment/run/tf-proxmox-template/main.tf b/deployment/run/tf-proxmox-template/main.tf new file mode 100644 index 00000000..5fb77695 --- /dev/null +++ b/deployment/run/tf-proxmox-template/main.tf @@ -0,0 +1,55 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "= 0.81.0" + } + } + backend "http" { + } +} + +locals { + dump_name = "qemu-nixos-fediversity-${var.category}.qcow2" +} + +# https://registry.terraform.io/providers/bpg/proxmox/latest/docs +provider "proxmox" { + endpoint = "https://${var.host}:8006/" + + # used for upload + ssh { + agent = true + username = "root" + } +} + +# hash of our code directory, used to trigger re-deploy +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ../../..)\\\"}\""] +} + +resource "proxmox_virtual_environment_file" "upload" { + depends_on = [ + data.external.hash, + ] + content_type = "import" + datastore_id = var.image_datastore_id + node_name = var.node_name + overwrite = true + timeout_upload = 500 + + source_file { + path = var.image + file_name = local.dump_name + checksum = var.checksum + } +} + +output "id" { + value = proxmox_virtual_environment_file.upload.id +} +output "path" { + value = proxmox_virtual_environment_file.upload.source_file[0].file_name +} diff --git a/deployment/run/tf-proxmox-template/run.sh b/deployment/run/tf-proxmox-template/run.sh new file mode 100644 index 00000000..a6941e31 --- /dev/null +++ b/deployment/run/tf-proxmox-template/run.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env bash +set -euo pipefail +declare tf_env + +cd "${tf_env}/deployment/run/tf-proxmox-template" +tofu apply --auto-approve -input=false -parallelism=1 >&2 +tofu output -json diff --git a/deployment/run/tf-proxmox-template/tf.nix b/deployment/run/tf-proxmox-template/tf.nix new file mode 100644 index 00000000..e123a729 --- /dev/null +++ b/deployment/run/tf-proxmox-template/tf.nix @@ -0,0 +1,48 @@ +# FIXME: use overlays so this gets imported just once? +{ + pkgs, +}: +# FIXME centralize overlays +# XXX using recent revision for https://github.com/NixOS/nixpkgs/pull/447849 +let + sources = import ../../../npins; + mkProvider = + args: + pkgs.terraform-providers.mkProvider ( + { mkProviderFetcher = { repo, ... }: sources.${repo}; } // args + ); +in +( + (pkgs.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/op/opentofu/package.nix" { }) + .overrideAttrs + (old: rec { + patches = (old.patches or [ ]) ++ [ + # TF with back-end poses a problem for nix: initialization involves both + # mutation (nix: only inside build) and a network call (nix: not inside build) + ../../check/data-model-tf/02-opentofu-sandboxed-init.patch + ]; + # versions > 1.9.0 need go 1.24+ + version = "1.9.0"; + src = pkgs.fetchFromGitHub { + owner = "opentofu"; + repo = "opentofu"; + tag = "v${version}"; + hash = "sha256-e0ZzbQdex0DD7Bj9WpcVI5roh0cMbJuNr5nsSVaOSu4="; + }; + vendorHash = "sha256-fMTbLSeW+pw6GK8/JLZzG2ER90ss2g1FSDX5+f292do="; + }) +).withPlugins + (p: [ + p.external + (mkProvider { + owner = "bpg"; + repo = "terraform-provider-proxmox"; + # 0.82+ need go 1.25 + rev = "v0.81.0"; + spdx = "MPL-2.0"; + hash = null; + vendorHash = "sha256-cpei22LkKqohlE76CQcIL5d7p+BjNcD6UQ8dl0WXUOc="; + homepage = "https://registry.terraform.io/providers/bpg/proxmox"; + provider-source-address = "registry.opentofu.org/bpg/proxmox"; + }) + ]) diff --git a/deployment/run/tf-proxmox-template/variables.tf b/deployment/run/tf-proxmox-template/variables.tf new file mode 100644 index 00000000..ec5ba791 --- /dev/null +++ b/deployment/run/tf-proxmox-template/variables.tf @@ -0,0 +1,31 @@ +variable "host" { + description = "the host of the ProxmoX Virtual Environment." + type = string +} + +variable "node_name" { + description = "the name of the ProxmoX node to use." + type = string +} + +variable "image" { + description = "Back-up file to upload." + type = string +} + +variable "image_datastore_id" { + description = "ID of the datastore of the image." + type = string + default = "local" +} + +variable "category" { + type = string + description = "Category to be used in naming the base image." + default = "test" +} + +variable "checksum" { + type = string + description = "The SHA256 checksum of the source file." +} diff --git a/deployment/run/tf-proxmox-vm/await-ssh.sh b/deployment/run/tf-proxmox-vm/await-ssh.sh new file mode 100644 index 00000000..1c83aee4 --- /dev/null +++ b/deployment/run/tf-proxmox-vm/await-ssh.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env bash +set -euo pipefail +declare username host key_file ssh_opts +readarray -t ssh_opts < <(echo "$ssh_opts" | jq -r '.[]') + +sshOpts=( + -o BatchMode=yes \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=5 \ + -o ServerAliveInterval=5 \ +) +if [[ -n "${key_file}" ]]; then + sshOpts+=( + -i "${key_file}" + ) +fi +for ssh_opt in "${ssh_opts[@]}"; do + sshOpts+=( + -o "${ssh_opt}" + ) +done + +for i in $(seq 1 30); do + if ssh "${sshOpts[@]}" "${username}@${host}" "true"; then + exit 0 + fi + echo "Waiting for SSH (attempt #$i)..." + sleep 5 +done +echo "SSH never came up!" >&2 +exit 1 diff --git a/deployment/run/tf-proxmox-vm/main.tf b/deployment/run/tf-proxmox-vm/main.tf new file mode 100644 index 00000000..dbfc5f10 --- /dev/null +++ b/deployment/run/tf-proxmox-vm/main.tf @@ -0,0 +1,138 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "= 0.81.0" + } + } + backend "http" { + } +} + +# https://registry.terraform.io/providers/bpg/proxmox/latest/docs +provider "proxmox" { + endpoint = "https://${var.host}:8006/" + + # used only for files and creating custom disks + # FIXME handle known-hosts in TF state + ssh { + agent = true + username = "root" + } +} + +# hash of our code directory, used to trigger re-deploy +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ../../..)\\\"}\""] +} + +resource "proxmox_virtual_environment_vm" "nix_vm" { + lifecycle { + ignore_changes = [ + disk["import_from"], + initialization, + ] + } + node_name = var.node_name + pool_id = var.pool_id + description = var.description + started = true + + # https://wiki.nixos.org/wiki/Virt-manager#Guest_Agent + agent { + enabled = true + timeout = "2m" + trim = true + } + + cpu { + type = "x86-64-v2-AES" + cores = var.cores + sockets = var.sockets + numa = true + } + + memory { + dedicated = var.memory + } + + disk { + datastore_id = var.vm_datastore_id + file_format = "qcow2" + interface = "scsi0" + discard = "on" + iothread = true + size = var.disk_size + ssd = true + backup = false + cache = "none" + import_from = var.template_id + } + + efi_disk { + datastore_id = var.vm_datastore_id + file_format = "qcow2" + type = "4m" + } + + network_device { + model = "virtio" + bridge = var.bridge + vlan_id = var.vlan_id + } + + operating_system { + type = "l26" + } + + scsi_hardware = "virtio-scsi-single" + bios = "ovmf" + + initialization { + datastore_id = var.cd_datastore_id + interface = "sata2" + ip_config { + ipv4 { + gateway = var.ipv4_gateway + address = var.ipv4_address + } + ipv6 { + gateway = var.ipv6_gateway + address = var.ipv6_address + } + } + } +} + +resource "null_resource" "await_ssh" { + depends_on = [ + proxmox_virtual_environment_vm.nix_vm + ] + provisioner "local-exec" { + command = "env username='root' host='${proxmox_virtual_environment_vm.nix_vm.ipv4_addresses[1][0]}' key_file='${var.key_file}' ssh_opts='${var.ssh_opts}' bash ./await-ssh.sh" + } +} + +module "nixos-rebuild" { + depends_on = [ + data.external.hash, + null_resource.await_ssh, + ] + source = "../tf-single-host" + nixos_conf = var.nixos_conf + username = "root" + host = proxmox_virtual_environment_vm.nix_vm.ipv4_addresses[1][0] + key_file = var.key_file + ssh_opts = var.ssh_opts +} + +output "id" { + value = proxmox_virtual_environment_vm.nix_vm.vm_id +} +output "ipv4" { + value = proxmox_virtual_environment_vm.nix_vm.ipv4_addresses[1] +} +output "ipv6" { + value = [ for elem in proxmox_virtual_environment_vm.nix_vm.ipv6_addresses[1] : "${elem}%${proxmox_virtual_environment_vm.nix_vm.network_interface_names[1]}" ] +} diff --git a/deployment/run/tf-proxmox-vm/run.sh b/deployment/run/tf-proxmox-vm/run.sh new file mode 100644 index 00000000..1bd46abd --- /dev/null +++ b/deployment/run/tf-proxmox-vm/run.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash +set -euo pipefail +declare tf_env + +cd "${tf_env}/deployment/run/tf-proxmox-vm" +# parallelism=1: limit OOM risk +tofu apply --auto-approve -input=false -parallelism=1 >&2 +tofu output -json diff --git a/deployment/run/tf-proxmox-vm/tf.nix b/deployment/run/tf-proxmox-vm/tf.nix new file mode 100644 index 00000000..bf4eea67 --- /dev/null +++ b/deployment/run/tf-proxmox-vm/tf.nix @@ -0,0 +1,49 @@ +# FIXME: use overlays so this gets imported just once? +{ + pkgs, +}: +# FIXME centralize overlays +# XXX using recent revision for https://github.com/NixOS/nixpkgs/pull/447849 +let + sources = import ../../../npins; + mkProvider = + args: + pkgs.terraform-providers.mkProvider ( + { mkProviderFetcher = { repo, ... }: sources.${repo}; } // args + ); +in +( + (pkgs.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/op/opentofu/package.nix" { }) + .overrideAttrs + (old: rec { + patches = (old.patches or [ ]) ++ [ + # TF with back-end poses a problem for nix: initialization involves both + # mutation (nix: only inside build) and a network call (nix: not inside build) + ../../check/data-model-tf/02-opentofu-sandboxed-init.patch + ]; + # versions > 1.9.0 need go 1.24+ + version = "1.9.0"; + src = pkgs.fetchFromGitHub { + owner = "opentofu"; + repo = "opentofu"; + tag = "v${version}"; + hash = "sha256-e0ZzbQdex0DD7Bj9WpcVI5roh0cMbJuNr5nsSVaOSu4="; + }; + vendorHash = "sha256-fMTbLSeW+pw6GK8/JLZzG2ER90ss2g1FSDX5+f292do="; + }) +).withPlugins + (p: [ + p.external + p.null + (mkProvider { + owner = "bpg"; + repo = "terraform-provider-proxmox"; + # 0.82+ need go 1.25 + rev = "v0.81.0"; + spdx = "MPL-2.0"; + hash = null; + vendorHash = "sha256-cpei22LkKqohlE76CQcIL5d7p+BjNcD6UQ8dl0WXUOc="; + homepage = "https://registry.terraform.io/providers/bpg/proxmox"; + provider-source-address = "registry.opentofu.org/bpg/proxmox"; + }) + ]) diff --git a/deployment/run/tf-proxmox-vm/variables.tf b/deployment/run/tf-proxmox-vm/variables.tf new file mode 100644 index 00000000..496cfaf0 --- /dev/null +++ b/deployment/run/tf-proxmox-vm/variables.tf @@ -0,0 +1,119 @@ +variable "nixos_conf" { + description = "The path to the NixOS configuration to deploy." + type = string +} + +variable "ssh_user" { + description = "the SSH user to use" + type = string + default = "root" +} + +variable "host" { + description = "the host of the ProxmoX Virtual Environment." + type = string +} + +variable "node_name" { + description = "the name of the ProxmoX node to use." + type = string +} + +variable "key_file" { + description = "path to the user's SSH private key" + type = string +} + +variable "ssh_opts" { + description = "Extra SSH options (`-o`) to use." + type = string + default = "[]" +} + +variable "bridge" { + description = "The name of the network bridge (defaults to vmbr0)." + type = string + default = "vmbr0" +} + +variable "vlan_id" { + description = "The VLAN identifier." + type = number + default = 0 +} + +variable "template_id" { + description = "ID of the template file from which to clone the VM." + type = string +} + +variable "vm_datastore_id" { + description = "ID of the datastore of the VM." + type = string + default = "local" +} + +variable "cd_datastore_id" { + description = "ID of the datastore of the virtual CD-rom drive to use for cloud-init." + type = string + default = "local" +} + +variable "ipv4_gateway" { + description = "Gateway for IPv4." + type = string + default = "" +} + +variable "ipv4_address" { + description = "IPv4 address." + type = string + default = "" +} + +variable "ipv6_gateway" { + description = "Gateway for IPv6." + type = string + default = "" +} + +variable "ipv6_address" { + description = "IPv6 address." + type = string + default = "" +} + +variable "description" { + type = string + default = "" +} + +variable "sockets" { + type = number + description = "The number of sockets of the VM." + default = 1 +} + +variable "cores" { + type = number + description = "The number of cores of the VM." + default = 1 +} + +variable "memory" { + type = number + description = "The amount of memory of the VM in MiB." + default = 2048 +} + +variable "disk_size" { + type = number + description = "The amount of disk of the VM in GiB." + default = 32 +} + +variable "pool_id" { + type = string + description = "The identifier for a pool to assign the virtual machine to." + default = "Fediversity" +} diff --git a/deployment/utils.nix b/deployment/utils.nix index 1229bf37..69345931 100644 --- a/deployment/utils.nix +++ b/deployment/utils.nix @@ -10,6 +10,7 @@ (lib.evalModules { specialArgs = { inherit pkgs inputs; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; }; modules = [ ./data-model.nix diff --git a/infra/common/proxmox-qemu-vm.nix b/infra/common/proxmox-qemu-vm.nix index 6b4970b3..78e113a7 100644 --- a/infra/common/proxmox-qemu-vm.nix +++ b/infra/common/proxmox-qemu-vm.nix @@ -40,6 +40,7 @@ priority = 1; size = "500M"; type = "EF00"; + label = "boot"; content = { type = "filesystem"; format = "vfat"; diff --git a/mkFlake.nix b/mkFlake.nix index bbcd6871..a388a908 100644 --- a/mkFlake.nix +++ b/mkFlake.nix @@ -26,7 +26,8 @@ let inputs = inputs'; }; - flake-parts-lib = import "${sources.flake-parts}/lib.nix" { inherit (nixpkgs) lib; }; + inherit (nixpkgs) lib outPath; + flake-parts-lib = import "${sources.flake-parts}/lib.nix" { inherit lib; }; in flakeModule: @@ -41,6 +42,7 @@ flake-parts-lib.mkFlake self = self'; specialArgs = { inherit sources; + modulesPath = "${builtins.toString outPath}/nixos/modules"; }; } { diff --git a/npins/sources.json b/npins/sources.json index 35452a17..3f7e07a9 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -202,9 +202,22 @@ }, "branch": "main", "submodules": false, - "revision": "48f39fbe2e8f90f9ac160dd4b6929f3ac06d8223", - "url": "https://github.com/SaumonNet/proxmox-nixos/archive/48f39fbe2e8f90f9ac160dd4b6929f3ac06d8223.tar.gz", - "hash": "0606qcs8x1jwckd1ivf52rqdmi3lkn66iiqh6ghd4kqx0g2bw3nv" + "revision": "ce8768f43b4374287cd8b88d8fa9c0061e749d9a", + "url": "https://github.com/SaumonNet/proxmox-nixos/archive/ce8768f43b4374287cd8b88d8fa9c0061e749d9a.tar.gz", + "hash": "116zplxh64wxbq81wsfkmmssjs1l228kvhxfi9d434xd54k6vr35" + }, + "terraform-provider-proxmox": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "bpg", + "repo": "terraform-provider-proxmox" + }, + "branch": "main", + "submodules": false, + "revision": "891066821bf7993a5006b12a44c5b36dbdb852d8", + "url": "https://github.com/bpg/terraform-provider-proxmox/archive/891066821bf7993a5006b12a44c5b36dbdb852d8.tar.gz", + "hash": "0nh1b1mgkycjib2hfzgmq142kgklnnhk4rci4339pfgqfi1z841a" } }, "version": 5