diff --git a/.forgejo/workflows/netbox-ips.yaml b/.forgejo/workflows/netbox-ips.yaml new file mode 100644 index 00000000..9e149e65 --- /dev/null +++ b/.forgejo/workflows/netbox-ips.yaml @@ -0,0 +1,22 @@ +name: netbox-ips + +on: + pull_request: + types: + - opened + - synchronize + - reopened + push: + branches: + - main + +concurrency: + cancel-in-progress: true + group: ${{ forgejo.workflow }}-${{ forgejo.event.pull_request.number || forgejo.ref }} + +jobs: + netbox-ips: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix build .#checks.x86_64-linux.netbox-ips -vL diff --git a/deployment/check/data-model-nixops4/nixosTest.nix b/deployment/check/data-model-nixops4/nixosTest.nix index 79497a05..401affbe 100644 --- a/deployment/check/data-model-nixops4/nixosTest.nix +++ b/deployment/check/data-model-nixops4/nixosTest.nix @@ -9,6 +9,7 @@ name = "deployment-model"; sourceFileset = lib.fileset.unions [ ../../data-model.nix + ../../run/default.nix ../../function.nix ../../utils.nix ../common/model.nix diff --git a/deployment/check/data-model-ssh/nixosTest.nix b/deployment/check/data-model-ssh/nixosTest.nix index f97ccaa1..6f7fb7e3 100644 --- a/deployment/check/data-model-ssh/nixosTest.nix +++ b/deployment/check/data-model-ssh/nixosTest.nix @@ -23,6 +23,7 @@ in name = "deployment-model"; sourceFileset = lib.fileset.unions [ ../../data-model.nix + ../../run/default.nix ../../function.nix ../../nixos.nix ../../run/ssh-single-host/run.sh diff --git a/deployment/check/netbox-ips/constants.nix b/deployment/check/netbox-ips/constants.nix new file mode 100644 index 00000000..8242c6fb --- /dev/null +++ b/deployment/check/netbox-ips/constants.nix @@ -0,0 +1,10 @@ +{ + targetMachines = [ + "node" + ]; + pathToRoot = builtins.path { + path = ../../..; + name = "root"; + }; + pathFromRoot = "/deployment/check/netbox-ips"; +} diff --git a/deployment/check/netbox-ips/default.nix b/deployment/check/netbox-ips/default.nix new file mode 100644 index 00000000..9fb576a6 --- /dev/null +++ b/deployment/check/netbox-ips/default.nix @@ -0,0 +1,55 @@ +{ + inputs, + sources, + system, +}: + +let + overlay = _: prev: { + terraform-backend = + prev.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/te/terraform-backend/package.nix" + { }; + # FIXME centralize overlays + # XXX using recent revision for https://github.com/NixOS/nixpkgs/pull/447849 + opentofu = + (pkgs.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/op/opentofu/package.nix" { }) + .overrideAttrs + (old: rec { + patches = (old.patches or [ ]) ++ [ + # TF with back-end poses a problem for nix: initialization involves both + # mutation (nix: only inside build) and a network call (nix: not inside build) + ../../check/data-model-tf/02-opentofu-sandboxed-init.patch + ]; + # versions > 1.9.0 need go 1.24+ + version = "1.9.0"; + src = pkgs.fetchFromGitHub { + owner = "opentofu"; + repo = "opentofu"; + tag = "v${version}"; + hash = "sha256-e0ZzbQdex0DD7Bj9WpcVI5roh0cMbJuNr5nsSVaOSu4="; + }; + vendorHash = "sha256-fMTbLSeW+pw6GK8/JLZzG2ER90ss2g1FSDX5+f292do="; + }); + }; + pkgs = import sources.nixpkgs { + inherit system; + overlays = [ overlay ]; + }; +in +pkgs.testers.runNixOSTest { + imports = [ + ../../data-model.nix + ../../function.nix + ../common/nixosTest.nix + ./nixosTest.nix + ]; + _module.args = { + inherit inputs sources; + modulesPath = "${builtins.toString pkgs.path}/nixos/modules"; + }; + inherit (import ./constants.nix) + targetMachines + pathToRoot + pathFromRoot + ; +} diff --git a/deployment/check/netbox-ips/nixosTest.nix b/deployment/check/netbox-ips/nixosTest.nix new file mode 100644 index 00000000..a14cd8a9 --- /dev/null +++ b/deployment/check/netbox-ips/nixosTest.nix @@ -0,0 +1,97 @@ +{ + lib, + pkgs, + sources, + ... +}: +let + inherit (pkgs) system; + inherit (pkgs.callPackage ../../utils.nix { }) evalOption; + backendPort = builtins.toString 8080; + tfBackend = fragment: { + address = "http://localhost:${backendPort}/state/${fragment}"; + }; + inherit + (pkgs.callPackage ../../run { + inherit sources system; + }) + tf-netbox-store-ips + tf-netbox-get-ip + ; + netbox-store-ips = evalOption "tf-netbox-store-ips" tf-netbox-store-ips { + httpBackend = tfBackend "proxmox-test/store-ips"; + startAddress = "192.168.10.236/24"; + endAddress = "192.168.10.240/24"; + }; + netbox-get-ip = evalOption "tf-netbox-get-ip" tf-netbox-get-ip { + httpBackend = tfBackend "proxmox-test/get-ip"; + }; + netboxUser = "netbox"; + netboxPassword = "netbox"; + changePassword = pkgs.writeText "change-password.py" '' + from users.models import User + u = User.objects.get(username='${netboxUser}') + u.set_password('${netboxPassword}') + u.save() + ''; +in +{ + _class = "nixosTest"; + name = "netbox-ips"; + + nodes.deployer = + { ... }: + { + imports = [ + ../../modules/terraform-backend + ]; + + nix.nixPath = [ + (lib.concatStringsSep ":" (lib.mapAttrsToList (k: v: k + "=" + v) sources)) + ]; + + environment.systemPackages = [ + pkgs.jq + (pkgs.callPackage ../../run/tf-netbox-store-ips/tf.nix { }) + (pkgs.callPackage ../../run/tf-netbox-get-ip/tf.nix { }) + ]; + + services.terraform-backend = { + enable = true; + settings = { + LISTEN_ADDR = ":${backendPort}"; + # FIXME randomly generate this + KMS_KEY = "tsjxw9NjKUBUlzbTnD7orqIAdEmpGYRARvxD51jtY+o="; + }; + }; + services.netbox = { + enable = true; + # FIXME randomly generate this + secretKeyFile = pkgs.writeText "netbox-secret" "634da8232803a8155a58584d3186127000207e079d600fc10a890e5cd59c2f4b8f0e0654005944d2ce87f5be9c22ceebec66"; + port = 8001; + }; + systemd.services.netbox.serviceConfig.TimeoutStartSec = "15m"; + }; + + extraTestScript = '' + deployer.succeed(""" + netbox-manage createsuperuser --noinput --user "${netboxUser}" --email "test@domain.tld" >&2 + cat '${changePassword}' | netbox-manage shell + """) + netbox_token = deployer.succeed(""" + curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" http://localhost:8001/api/users/tokens/provision/ --data '{"username":"${netboxUser}","password":"${netboxPassword}"}' | jq -r .key + """).strip() + ip_range_id = deployer.succeed(f""" + export NETBOX_SERVER_URL="localhost:8001" + export NETBOX_API_TOKEN="{netbox_token}" + ${lib.getExe netbox-store-ips.run} | jq -r '.id.value' + """).strip() + ipv4 = deployer.succeed(f""" + export NETBOX_SERVER_URL="localhost:8001" + export NETBOX_API_TOKEN="{netbox_token}" + export TF_VAR_ip_range_id={ip_range_id} + ${lib.getExe netbox-get-ip.run} | jq -r '.ipv4.value' + """).strip() + assert ipv4 == "192.168.10.236/24" + ''; +} diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 3efe927d..ae400985 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -13,38 +13,9 @@ let attrsOf deferredModuleWith functionTo - nullOr optionType - raw - str submodule ; - inherit (pkgs.callPackage ./utils.nix { }) toBash withPackages tfApply; - writeConfig = - { - system, - caller, - root-path, - deployment-type, - deployment-name, - args, - }: - # having a `caller` location and (serializable) `args`, we know - # enough to call it again to extract different info elsewhere later. - # we use this to make a deployment script using the desired nixos config, - # which would otherwise not be serializable, while nix also makes it hard to - # produce its derivation to pass thru without a `nix-instantiate` call, - # which in turn would need to be passed the (unserializable) nixos config. - builtins.toString ( - pkgs.writers.writeText "configuration.nix" '' - import ${root-path}/deployment/nixos.nix { - system = "${system}"; - configuration = (import "${root-path}/${caller}" (builtins.fromJSON "${ - lib.replaceStrings [ "\"" ] [ "\\\"" ] (lib.strings.toJSON args) - }")).${deployment-name}.${deployment-type}.nixos-configuration; - } - '' - ); functionType = submodule ./function.nix; application-resources = submodule { @@ -57,538 +28,7 @@ let ); }; }; - nixops4Deployment = types.deferredModuleWith { - staticModules = [ - inputs.nixops4.modules.nixops4Deployment.default - - { - _class = "nixops4Deployment"; - _module.args = { - resourceProviderSystem = pkgs.system; - resources = { }; - }; - } - ]; - }; - nixos-configuration = mkOption { - description = "A NixOS configuration."; - type = raw; - }; - httpBackend = mkOption { - description = "environment variables to configure the TF HTTP back-end, see "; - type = types.submodule (http-backend: { - options = { - value = mkOption { - readOnly = true; - default = lib.mapAttrs' (k: v: lib.nameValuePair "TF_HTTP_${lib.toUpper k}" (builtins.toString v)) { - inherit (http-backend.config) - address - update_method - lock_address - lock_method - unlock_address - unlock_method - username - password - skip_cert_verification - retry_max - retry_wait_min - retry_wait_max - ; - }; - }; - address = mkOption { - description = "The address of the REST endpoint"; - type = str; - }; - update_method = mkOption { - description = "HTTP method to use when updating state."; - type = str; - default = "POST"; - }; - lock_address = mkOption { - description = "The address of the lock REST endpoint."; - type = str; - default = http-backend.config.address; - }; - lock_method = mkOption { - description = "The HTTP method to use when locking."; - type = str; - default = "LOCK"; - }; - unlock_address = mkOption { - description = "The address of the unlock REST endpoint."; - type = str; - default = http-backend.config.address; - }; - unlock_method = mkOption { - description = "The HTTP method to use when unlocking."; - type = str; - default = "UNLOCK"; - }; - username = mkOption { - description = "The username for HTTP basic authentication."; - type = str; - default = "basic"; - }; - password = mkOption { - description = "The password for HTTP basic authentication."; - type = str; - default = "fake-secret"; - }; - skip_cert_verification = mkOption { - description = "Whether to skip TLS verification."; - type = str; - default = "false"; - }; - retry_max = mkOption { - description = "The number of HTTP request retries."; - type = types.int; - default = 2; - }; - retry_wait_min = mkOption { - description = "The minimum time in seconds to wait between HTTP request attempts."; - type = types.int; - default = 1; - }; - retry_wait_max = mkOption { - description = "The maximum time in seconds to wait between HTTP request attempts."; - type = types.int; - default = 30; - }; - }; - }); - }; - host-ssh = mkOption { - description = "SSH connection info to connect to a single host."; - type = submodule { - options = { - host = mkOption { - description = "the host to access by SSH"; - type = str; - }; - username = mkOption { - description = "the SSH user to use"; - type = nullOr str; - default = null; - }; - key-file = mkOption { - description = "path to the user's SSH private key"; - type = nullOr str; - example = "/root/.ssh/id_ed25519"; - }; - sshOpts = mkOption { - description = "Extra SSH options (`-o`) to use."; - type = types.listOf str; - default = [ ]; - example = "ConnectTimeout=60"; - }; - }; - }; - }; - # FIXME allow custom deployment types - # FIXME make deployments environment resources? - deployment-type = attrTag { - ssh-host = mkOption { - description = "A deployment by SSH to update a single existing NixOS host."; - type = submodule (ssh-host: { - options = { - system = mkOption { - description = "The architecture of the system to deploy to."; - type = types.str; - }; - inherit nixos-configuration; - ssh = host-ssh; - caller = mkOption { - description = "The calling module to obtain the NixOS configuration from."; - type = types.str; - }; - args = mkOption { - description = "The arguments with which to call the module to obtain the NixOS configuration."; - type = types.attrs; - }; - deployment-name = mkOption { - description = "The name of the deployment for which to obtain the NixOS configuration."; - type = types.str; - default = "default"; - }; - root-path = mkOption { - description = "The path to the root of the repository."; - type = types.path; - }; - run = mkOption { - type = types.package; - # error: The option `ssh-deployment.ssh-host.run' is read-only, but it's set multiple times. - # readOnly = true; - default = - let - inherit (ssh-host.config) - system - ssh - caller - args - deployment-name - root-path - ; - inherit (ssh) - host - username - key-file - sshOpts - ; - environment = { - key_file = key-file; - ssh_opts = sshOpts; - inherit - host - username - ; - nixos_conf = writeConfig { - inherit - system - caller - args - deployment-name - root-path - ; - deployment-type = "ssh-host"; - }; - }; - in - pkgs.writers.writeBashBin "deploy-sh.sh" - (withPackages [ - pkgs.jq - ]) - '' - env ${ - toString (lib.mapAttrsToList (k: v: "${k}=\"${toBash v}\"") environment) - } bash ./deployment/run/ssh-single-host/run.sh - ''; - }; - }; - }); - }; - nixops4 = mkOption { - description = "A NixOps4 NixOS deployment. For an example, see https://github.com/nixops4/nixops4-nixos/blob/main/example/deployment.nix."; - type = nixops4Deployment; - }; - tf-host = mkOption { - description = "A Terraform deployment by SSH to update a single existing NixOS host."; - type = submodule (tf-host: { - options = { - system = mkOption { - description = "The architecture of the system to deploy to."; - type = types.str; - }; - inherit httpBackend nixos-configuration; - ssh = host-ssh; - caller = mkOption { - description = "The calling module to obtain the NixOS configuration from."; - type = types.str; - }; - args = mkOption { - description = "The arguments with which to call the module to obtain the NixOS configuration."; - type = types.attrs; - }; - deployment-name = mkOption { - description = "The name of the deployment for which to obtain the NixOS configuration."; - type = types.str; - }; - root-path = mkOption { - description = "The path to the root of the repository."; - type = types.path; - }; - run = mkOption { - type = types.package; - # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. - # readOnly = true; - default = - let - inherit (tf-host.config) - system - ssh - caller - args - deployment-name - root-path - httpBackend - ; - inherit (ssh) - host - username - key-file - sshOpts - ; - in - tfApply { - inherit httpBackend; - directory = "tf-single-host"; - environment = { - key_file = key-file; - ssh_opts = sshOpts; - inherit - host - username - ; - nixos_conf = writeConfig { - inherit - system - caller - args - deployment-name - root-path - ; - deployment-type = "tf-host"; - }; - }; - }; - }; - }; - }); - }; - tf-proxmox-template = mkOption { - description = '' - A Terraform deployment to upload a virtual machine template to ProxmoX VE. - Proxmox credentials should be set using [environment variables] - (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) - with role `PVEDatastoreAdmin`. - ''; - type = submodule (tf-host: { - options = { - system = mkOption { - description = "The architecture of the system to deploy to."; - type = types.str; - }; - inherit httpBackend nixos-configuration; - ssh = host-ssh; - node-name = mkOption { - description = "the name of the ProxmoX node to use."; - type = types.str; - }; - imageDatastoreId = mkOption { - description = "ID of the datastore of the image."; - type = types.str; - default = "local"; - }; - run = mkOption { - type = types.package; - # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. - # readOnly = true; - default = - let - inherit (tf-host.config) - system - ssh - httpBackend - node-name - imageDatastoreId - ; - inherit (ssh) - host - ; - machine = import ./nixos.nix { - inherit sources system; - configuration = tf-host.config.nixos-configuration; - }; - name = "fediversity-template"; - - # worse for cross-compilation, better for pre-/post-processing, needs manual `imageSize`, random failures: https://github.com/nix-community/disko/issues/550#issuecomment-2503736973 - raw = "${machine.config.system.build.diskoImages}/main.raw"; - - environment = { - inherit - host - ; - node_name = node-name; - image_datastore_id = imageDatastoreId; - }; - in - lib.trace (lib.strings.toJSON environment) pkgs.writers.writeBashBin "deploy-tf-proxmox-template.sh" - (withPackages [ - pkgs.qemu - ]) - '' - set -e - - # nixos-generate gives the burden of building revisions, while systemd-repart handles partitioning ~~at the burden of version revisions~~ - # .qcow2 is around half the size of .raw, on top of supporting backups - be it apparently at the cost of performance - qemu-img convert -f raw -O qcow2 -C "${raw}" /tmp/${name}.qcow2 - - ls -l ${raw} >&2 - ls -l /tmp/${name}.qcow2 >&2 - checksum="$(sha256sum /tmp/${name}.qcow2 | cut -d " " -f1)" - - env \ - TF_VAR_image=/tmp/${name}.qcow2 \ - TF_VAR_checksum="$checksum" \ - ${lib.getExe (tfApply { - inherit httpBackend environment; - directory = "tf-proxmox-template"; - })} - ''; - }; - }; - }); - }; - tf-proxmox-vm = mkOption { - description = '' - A Terraform deployment to provision and update a virtual machine on ProxmoX VE. - Proxmox credentials should be set using [environment variables] - (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) - with roles `PVEVMAdmin PVEDatastoreAdmin PVESDNUser`. - ''; - type = submodule (tf-host: { - options = { - system = mkOption { - description = "The architecture of the system to deploy to."; - type = types.str; - }; - inherit httpBackend nixos-configuration; - ssh = host-ssh; - caller = mkOption { - description = "The calling module to obtain the NixOS configuration from."; - type = types.str; - }; - args = mkOption { - description = "The arguments with which to call the module to obtain the NixOS configuration."; - type = types.attrs; - }; - deployment-name = mkOption { - description = "The name of the deployment for which to obtain the NixOS configuration."; - type = types.str; - }; - root-path = mkOption { - description = "The path to the root of the repository."; - type = types.path; - }; - node-name = mkOption { - description = "the name of the ProxmoX node to use."; - type = types.str; - }; - bridge = mkOption { - description = "The name of the network bridge (defaults to vmbr0)."; - type = types.str; - default = "vmbr0"; - }; - vlanId = mkOption { - description = "The VLAN identifier."; - type = types.int; - default = 0; - }; - imageDatastoreId = mkOption { - description = "ID of the datastore of the image."; - type = types.str; - default = "local"; - }; - templateId = mkOption { - description = "ID of the template file from which to clone the VM."; - type = types.nullOr types.str; - example = "local:import/template.qcow2"; - }; - vmDatastoreId = mkOption { - description = "ID of the datastore of the VM."; - type = types.str; - default = "local"; - }; - cdDatastoreId = mkOption { - description = "ID of the datastore of the virtual CD-rom drive to use for cloud-init."; - type = types.str; - default = "local"; - }; - ipv4Gateway = mkOption { - description = "Gateway for IPv4."; - type = types.str; - default = ""; - }; - ipv4Address = mkOption { - description = "IPv4 address."; - type = types.str; - default = ""; - }; - ipv6Gateway = mkOption { - description = "Gateway for IPv6."; - type = types.str; - default = ""; - }; - ipv6Address = mkOption { - description = "IPv6 address."; - type = types.str; - default = ""; - }; - run = mkOption { - type = types.package; - # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. - # readOnly = true; - default = - let - inherit (tf-host.config) - system - ssh - caller - args - deployment-name - httpBackend - root-path - node-name - bridge - vlanId - imageDatastoreId - templateId - vmDatastoreId - cdDatastoreId - ipv4Gateway - ipv4Address - ipv6Gateway - ipv6Address - ; - inherit (ssh) - host - username - key-file - sshOpts - ; - deployment-type = "tf-proxmox-vm"; - nixos_conf = writeConfig { - inherit - system - caller - args - deployment-name - root-path - deployment-type - ; - }; - environment = { - key_file = key-file; - ssh_opts = sshOpts; - inherit - host - nixos_conf - bridge - ; - node_name = node-name; - ssh_user = username; - vlan_id = vlanId; - image_datastore_id = imageDatastoreId; - template_id = templateId; - vm_datastore_id = vmDatastoreId; - cd_datastore_id = cdDatastoreId; - ipv4_gateway = ipv4Gateway; - ipv4_address = ipv4Address; - ipv6_gateway = ipv6Gateway; - ipv6_address = ipv6Address; - }; - in - lib.trace (lib.strings.toJSON environment) (tfApply { - inherit httpBackend environment; - directory = "tf-proxmox-vm"; - dependentDirs = [ "tf-single-host" ]; - }); - }; - }; - }); - }; - }; + deployment-type = attrTag (pkgs.callPackage ./run { inherit inputs sources; }); in { options = { diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 002e71c8..f9272a19 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -44,6 +44,10 @@ deployment-model-tf-proxmox = import ./check/data-model-tf-proxmox { inherit inputs sources system; }; + + netbox-ips = import ./check/netbox-ips { + inherit inputs sources system; + }; }; }; } diff --git a/deployment/run/default.nix b/deployment/run/default.nix new file mode 100644 index 00000000..942fae96 --- /dev/null +++ b/deployment/run/default.nix @@ -0,0 +1,635 @@ +{ + pkgs, + lib, + sources ? import ../../npins, + inputs ? null, + ... +}: +# FIXME allow custom deployment types +# FIXME make deployments environment resources? +let + inherit (lib) mkOption types; + inherit (lib.types) + nullOr + raw + str + submodule + ; + inherit (pkgs.callPackage ../utils.nix { }) toBash withPackages tfApply; + writeConfig = + { + system, + caller, + root-path, + deployment-type, + deployment-name, + args, + }: + # having a `caller` location and (serializable) `args`, we know + # enough to call it again to extract different info elsewhere later. + # we use this to make a deployment script using the desired nixos config, + # which would otherwise not be serializable, while nix also makes it hard to + # produce its derivation to pass thru without a `nix-instantiate` call, + # which in turn would need to be passed the (unserializable) nixos config. + builtins.toString ( + pkgs.writers.writeText "configuration.nix" '' + import ${root-path}/deployment/nixos.nix { + system = "${system}"; + configuration = (import "${root-path}/${caller}" (builtins.fromJSON "${ + lib.replaceStrings [ "\"" ] [ "\\\"" ] (lib.strings.toJSON args) + }")).${deployment-name}.${deployment-type}.nixos-configuration; + } + '' + ); + nixops4Deployment = types.deferredModuleWith { + staticModules = [ + inputs.nixops4.modules.nixops4Deployment.default + + { + _class = "nixops4Deployment"; + _module.args = { + resourceProviderSystem = pkgs.system; + resources = { }; + }; + } + ]; + }; + nixos-configuration = mkOption { + description = "A NixOS configuration."; + type = raw; + }; + httpBackend = mkOption { + description = "environment variables to configure the TF HTTP back-end, see "; + # type = types.attrsOf (types.either types.str types.int); + type = types.submodule (http-backend: { + options = { + value = mkOption { + readOnly = true; + default = lib.mapAttrs' (k: v: lib.nameValuePair "TF_HTTP_${lib.toUpper k}" (builtins.toString v)) { + inherit (http-backend.config) + address + update_method + lock_address + lock_method + unlock_address + unlock_method + username + password + skip_cert_verification + retry_max + retry_wait_min + retry_wait_max + ; + }; + }; + address = mkOption { + description = "The address of the REST endpoint"; + type = str; + }; + update_method = mkOption { + description = "HTTP method to use when updating state."; + type = str; + default = "POST"; + }; + lock_address = mkOption { + description = "The address of the lock REST endpoint."; + type = str; + default = http-backend.config.address; + }; + lock_method = mkOption { + description = "The HTTP method to use when locking."; + type = str; + default = "LOCK"; + }; + unlock_address = mkOption { + description = "The address of the unlock REST endpoint."; + type = str; + default = http-backend.config.address; + }; + unlock_method = mkOption { + description = "The HTTP method to use when unlocking."; + type = str; + default = "UNLOCK"; + }; + username = mkOption { + description = "The username for HTTP basic authentication."; + type = str; + default = "basic"; + }; + password = mkOption { + description = "The password for HTTP basic authentication."; + type = str; + default = "fake-secret"; + }; + skip_cert_verification = mkOption { + description = "Whether to skip TLS verification."; + type = str; + default = "false"; + }; + retry_max = mkOption { + description = "The number of HTTP request retries."; + type = types.int; + default = 2; + }; + retry_wait_min = mkOption { + description = "The minimum time in seconds to wait between HTTP request attempts."; + type = types.int; + default = 1; + }; + retry_wait_max = mkOption { + description = "The maximum time in seconds to wait between HTTP request attempts."; + type = types.int; + default = 30; + }; + }; + }); + }; + host-ssh = mkOption { + description = "SSH connection info to connect to a single host."; + type = submodule { + options = { + host = mkOption { + description = "the host to access by SSH"; + type = str; + }; + username = mkOption { + description = "the SSH user to use"; + type = nullOr str; + default = null; + }; + key-file = mkOption { + description = "path to the user's SSH private key"; + type = nullOr str; + example = "/root/.ssh/id_ed25519"; + }; + sshOpts = mkOption { + description = "Extra SSH options (`-o`) to use."; + type = types.listOf str; + default = [ ]; + example = "ConnectTimeout=60"; + }; + }; + }; + }; +in +{ + ssh-host = mkOption { + description = "A deployment by SSH to update a single existing NixOS host."; + type = submodule (ssh-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit nixos-configuration; + ssh = host-ssh; + caller = mkOption { + description = "The calling module to obtain the NixOS configuration from."; + type = types.str; + }; + args = mkOption { + description = "The arguments with which to call the module to obtain the NixOS configuration."; + type = types.attrs; + }; + deployment-name = mkOption { + description = "The name of the deployment for which to obtain the NixOS configuration."; + type = types.str; + default = "default"; + }; + root-path = mkOption { + description = "The path to the root of the repository."; + type = types.path; + }; + run = mkOption { + type = types.package; + # error: The option `ssh-deployment.ssh-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (ssh-host.config) + system + ssh + caller + args + deployment-name + root-path + ; + inherit (ssh) + host + username + key-file + sshOpts + ; + environment = { + key_file = key-file; + ssh_opts = sshOpts; + inherit + host + username + ; + nixos_conf = writeConfig { + inherit + system + caller + args + deployment-name + root-path + ; + deployment-type = "ssh-host"; + }; + }; + in + pkgs.writers.writeBashBin "deploy-sh.sh" + (withPackages [ + pkgs.jq + ]) + '' + env ${ + toString (lib.mapAttrsToList (k: v: "${k}=\"${toBash v}\"") environment) + } bash ./deployment/run/ssh-single-host/run.sh + ''; + }; + }; + }); + }; + nixops4 = mkOption { + description = "A NixOps4 NixOS deployment. For an example, see https://github.com/nixops4/nixops4-nixos/blob/main/example/deployment.nix."; + type = nixops4Deployment; + }; + tf-host = mkOption { + description = "A Terraform deployment by SSH to update a single existing NixOS host."; + type = submodule (tf-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit httpBackend nixos-configuration; + ssh = host-ssh; + caller = mkOption { + description = "The calling module to obtain the NixOS configuration from."; + type = types.str; + }; + args = mkOption { + description = "The arguments with which to call the module to obtain the NixOS configuration."; + type = types.attrs; + }; + deployment-name = mkOption { + description = "The name of the deployment for which to obtain the NixOS configuration."; + type = types.str; + }; + root-path = mkOption { + description = "The path to the root of the repository."; + type = types.path; + }; + run = mkOption { + type = types.package; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + caller + args + deployment-name + root-path + httpBackend + ; + inherit (ssh) + host + username + key-file + sshOpts + ; + in + tfApply { + inherit httpBackend; + directory = "tf-single-host"; + environment = { + key_file = key-file; + ssh_opts = sshOpts; + inherit + host + username + ; + nixos_conf = writeConfig { + inherit + system + caller + args + deployment-name + root-path + ; + deployment-type = "tf-host"; + }; + }; + }; + }; + }; + }); + }; + tf-proxmox-template = mkOption { + description = '' + A Terraform deployment to upload a virtual machine template to ProxmoX VE. + Proxmox credentials should be set using [environment variables] + (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) + with role `PVEDatastoreAdmin`. + ''; + type = submodule (tf-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit httpBackend nixos-configuration; + ssh = host-ssh; + node-name = mkOption { + description = "the name of the ProxmoX node to use."; + type = types.str; + }; + imageDatastoreId = mkOption { + description = "ID of the datastore of the image."; + type = types.str; + default = "local"; + }; + run = mkOption { + type = types.package; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + httpBackend + node-name + imageDatastoreId + ; + inherit (ssh) + host + ; + machine = import ../nixos.nix { + inherit sources system; + configuration = tf-host.config.nixos-configuration; + }; + name = "fediversity-template"; + + # worse for cross-compilation, better for pre-/post-processing, needs manual `imageSize`, random failures: https://github.com/nix-community/disko/issues/550#issuecomment-2503736973 + raw = "${machine.config.system.build.diskoImages}/main.raw"; + + environment = { + inherit + host + ; + node_name = node-name; + image_datastore_id = imageDatastoreId; + }; + in + lib.trace (lib.strings.toJSON environment) pkgs.writers.writeBashBin "deploy-tf-proxmox-template.sh" + (withPackages [ + pkgs.qemu + ]) + '' + set -e + + # nixos-generate gives the burden of building revisions, while systemd-repart handles partitioning ~~at the burden of version revisions~~ + # .qcow2 is around half the size of .raw, on top of supporting backups - be it apparently at the cost of performance + qemu-img convert -f raw -O qcow2 -C "${raw}" /tmp/${name}.qcow2 + + ls -l ${raw} >&2 + ls -l /tmp/${name}.qcow2 >&2 + checksum="$(sha256sum /tmp/${name}.qcow2 | cut -d " " -f1)" + + env \ + TF_VAR_image=/tmp/${name}.qcow2 \ + TF_VAR_checksum="$checksum" \ + ${lib.getExe (tfApply { + inherit httpBackend environment; + directory = "tf-proxmox-template"; + })} + ''; + }; + }; + }); + }; + tf-proxmox-vm = mkOption { + description = '' + A Terraform deployment to provision and update a virtual machine on ProxmoX VE. + Proxmox credentials should be set using [environment variables] + (https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) + with roles `PVEVMAdmin PVEDatastoreAdmin PVESDNUser`. + ''; + type = submodule (tf-host: { + options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; + inherit httpBackend nixos-configuration; + ssh = host-ssh; + caller = mkOption { + description = "The calling module to obtain the NixOS configuration from."; + type = types.str; + }; + args = mkOption { + description = "The arguments with which to call the module to obtain the NixOS configuration."; + type = types.attrs; + }; + deployment-name = mkOption { + description = "The name of the deployment for which to obtain the NixOS configuration."; + type = types.str; + }; + root-path = mkOption { + description = "The path to the root of the repository."; + type = types.path; + }; + node-name = mkOption { + description = "the name of the ProxmoX node to use."; + type = types.str; + }; + bridge = mkOption { + description = "The name of the network bridge (defaults to vmbr0)."; + type = types.str; + default = "vmbr0"; + }; + vlanId = mkOption { + description = "The VLAN identifier."; + type = types.int; + default = 0; + }; + imageDatastoreId = mkOption { + description = "ID of the datastore of the image."; + type = types.str; + default = "local"; + }; + templateId = mkOption { + description = "ID of the template file from which to clone the VM."; + type = types.nullOr types.str; + example = "local:import/template.qcow2"; + }; + vmDatastoreId = mkOption { + description = "ID of the datastore of the VM."; + type = types.str; + default = "local"; + }; + cdDatastoreId = mkOption { + description = "ID of the datastore of the virtual CD-rom drive to use for cloud-init."; + type = types.str; + default = "local"; + }; + ipv4Gateway = mkOption { + description = "Gateway for IPv4."; + type = types.str; + default = ""; + }; + ipv4Address = mkOption { + description = "IPv4 address."; + type = types.nullOr types.str; + default = ""; + }; + ipv6Gateway = mkOption { + description = "Gateway for IPv6."; + type = types.str; + default = ""; + }; + ipv6Address = mkOption { + description = "IPv6 address."; + type = types.str; + default = ""; + }; + run = mkOption { + type = types.package; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + caller + args + deployment-name + httpBackend + root-path + node-name + bridge + vlanId + imageDatastoreId + templateId + vmDatastoreId + cdDatastoreId + ipv4Gateway + ipv4Address + ipv6Gateway + ipv6Address + ; + inherit (ssh) + host + username + key-file + sshOpts + ; + deployment-type = "tf-proxmox-vm"; + nixos_conf = writeConfig { + inherit + system + caller + args + deployment-name + root-path + deployment-type + ; + }; + environment = { + key_file = key-file; + ssh_opts = sshOpts; + inherit + host + nixos_conf + bridge + ; + node_name = node-name; + ssh_user = username; + vlan_id = vlanId; + image_datastore_id = imageDatastoreId; + template_id = templateId; + vm_datastore_id = vmDatastoreId; + cd_datastore_id = cdDatastoreId; + ipv4_gateway = ipv4Gateway; + ipv4_address = ipv4Address; + ipv6_gateway = ipv6Gateway; + ipv6_address = ipv6Address; + }; + in + lib.trace (lib.strings.toJSON environment) (tfApply { + inherit httpBackend environment; + directory = "tf-proxmox-vm"; + dependentDirs = [ "tf-single-host" ]; + }); + }; + }; + }); + }; + tf-netbox-store-ips = mkOption { + description = "Store a range of IPs in a Netbox instance."; + type = submodule (tf-netbox-store-ips: { + options = { + inherit httpBackend; + startAddress = mkOption { + description = "Start of the IP range."; + type = types.str; + example = "10.0.0.1/24"; + }; + endAddress = mkOption { + description = "End of the IP range."; + type = types.str; + example = "10.0.0.50/24"; + }; + run = mkOption { + type = types.package; + default = + let + inherit (tf-netbox-store-ips.config) + httpBackend + startAddress + endAddress + ; + in + tfApply { + inherit httpBackend; + directory = "tf-netbox-store-ips"; + environment = { + start_address = startAddress; + end_address = endAddress; + }; + }; + }; + }; + }); + }; + tf-netbox-get-ip = mkOption { + description = "Get an available IP from a Netbox instance."; + type = submodule (tf-netbox-get-ip: { + options = { + inherit httpBackend; + run = mkOption { + type = types.package; + default = + let + inherit (tf-netbox-get-ip.config) + httpBackend + ; + in + tfApply { + inherit httpBackend; + directory = "tf-netbox-get-ip"; + environment = { + }; + }; + }; + }; + }); + }; +} diff --git a/deployment/run/tf-netbox-get-ip/main.tf b/deployment/run/tf-netbox-get-ip/main.tf new file mode 100644 index 00000000..24f8eb9e --- /dev/null +++ b/deployment/run/tf-netbox-get-ip/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + netbox = { + source = "e-breuninger/netbox" + version = "= 5.0.0" + } + } + backend "http" { + } +} + +resource "netbox_available_ip_address" "get_ip" { + prefix_id = var.prefix_id + ip_range_id = var.ip_range_id +} + +output "ipv4" { + value = netbox_available_ip_address.get_ip.ip_address +} diff --git a/deployment/run/tf-netbox-get-ip/tf.nix b/deployment/run/tf-netbox-get-ip/tf.nix new file mode 100644 index 00000000..115d5cbe --- /dev/null +++ b/deployment/run/tf-netbox-get-ip/tf.nix @@ -0,0 +1,47 @@ +# FIXME: use overlays so this gets imported just once? +{ + pkgs, +}: +# FIXME centralize overlays +# XXX using recent revision for https://github.com/NixOS/nixpkgs/pull/447849 +let + sources = import ../../../npins; + mkProvider = + args: + pkgs.terraform-providers.mkProvider ( + { mkProviderFetcher = { repo, ... }: sources.${repo}; } // args + ); +in +( + (pkgs.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/op/opentofu/package.nix" { }) + .overrideAttrs + (old: rec { + patches = (old.patches or [ ]) ++ [ + # TF with back-end poses a problem for nix: initialization involves both + # mutation (nix: only inside build) and a network call (nix: not inside build) + ../../check/data-model-tf/02-opentofu-sandboxed-init.patch + ]; + # versions > 1.9.0 need go 1.24+ + version = "1.9.0"; + src = pkgs.fetchFromGitHub { + owner = "opentofu"; + repo = "opentofu"; + tag = "v${version}"; + hash = "sha256-e0ZzbQdex0DD7Bj9WpcVI5roh0cMbJuNr5nsSVaOSu4="; + }; + vendorHash = "sha256-fMTbLSeW+pw6GK8/JLZzG2ER90ss2g1FSDX5+f292do="; + }) +).withPlugins + (_: [ + (mkProvider { + owner = "e-breuninger"; + repo = "terraform-provider-netbox"; + rev = "v5.0.0"; + spdx = "MPL-2.0"; + # hash = "sha256-iCaCt8ZbkxCk43QEyj3PeHYuKPCPVU2oQ78aumH/l6k="; + hash = null; + vendorHash = "sha256-Q3H/6mpkWn1Gw0NRMtKtkBRGHjPJZGBFdGwfalyQ4Z0="; + homepage = "https://registry.terraform.io/providers/e-breuninger/netbox"; + provider-source-address = "registry.opentofu.org/e-breuninger/netbox"; + }) + ]) diff --git a/deployment/run/tf-netbox-get-ip/variables.tf b/deployment/run/tf-netbox-get-ip/variables.tf new file mode 100644 index 00000000..d46e238c --- /dev/null +++ b/deployment/run/tf-netbox-get-ip/variables.tf @@ -0,0 +1,11 @@ +variable "prefix_id" { + description = "The `id` output of a prefix resource / data source." + type = number + default = null +} + +variable "ip_range_id" { + description = "The `id` output of a `netbox_ip_range` resource." + type = number + default = null +} diff --git a/deployment/run/tf-netbox-store-ips/main.tf b/deployment/run/tf-netbox-store-ips/main.tf new file mode 100644 index 00000000..7d2ffa86 --- /dev/null +++ b/deployment/run/tf-netbox-store-ips/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + netbox = { + source = "e-breuninger/netbox" + version = "= 5.0.0" + } + } + backend "http" { + } +} + +resource "netbox_ip_range" "ips" { + start_address = var.start_address + end_address = var.end_address +} + +output "id" { + value = netbox_ip_range.ips.id +} diff --git a/deployment/run/tf-netbox-store-ips/tf.nix b/deployment/run/tf-netbox-store-ips/tf.nix new file mode 100644 index 00000000..115d5cbe --- /dev/null +++ b/deployment/run/tf-netbox-store-ips/tf.nix @@ -0,0 +1,47 @@ +# FIXME: use overlays so this gets imported just once? +{ + pkgs, +}: +# FIXME centralize overlays +# XXX using recent revision for https://github.com/NixOS/nixpkgs/pull/447849 +let + sources = import ../../../npins; + mkProvider = + args: + pkgs.terraform-providers.mkProvider ( + { mkProviderFetcher = { repo, ... }: sources.${repo}; } // args + ); +in +( + (pkgs.callPackage "${sources.nixpkgs-unstable}/pkgs/by-name/op/opentofu/package.nix" { }) + .overrideAttrs + (old: rec { + patches = (old.patches or [ ]) ++ [ + # TF with back-end poses a problem for nix: initialization involves both + # mutation (nix: only inside build) and a network call (nix: not inside build) + ../../check/data-model-tf/02-opentofu-sandboxed-init.patch + ]; + # versions > 1.9.0 need go 1.24+ + version = "1.9.0"; + src = pkgs.fetchFromGitHub { + owner = "opentofu"; + repo = "opentofu"; + tag = "v${version}"; + hash = "sha256-e0ZzbQdex0DD7Bj9WpcVI5roh0cMbJuNr5nsSVaOSu4="; + }; + vendorHash = "sha256-fMTbLSeW+pw6GK8/JLZzG2ER90ss2g1FSDX5+f292do="; + }) +).withPlugins + (_: [ + (mkProvider { + owner = "e-breuninger"; + repo = "terraform-provider-netbox"; + rev = "v5.0.0"; + spdx = "MPL-2.0"; + # hash = "sha256-iCaCt8ZbkxCk43QEyj3PeHYuKPCPVU2oQ78aumH/l6k="; + hash = null; + vendorHash = "sha256-Q3H/6mpkWn1Gw0NRMtKtkBRGHjPJZGBFdGwfalyQ4Z0="; + homepage = "https://registry.terraform.io/providers/e-breuninger/netbox"; + provider-source-address = "registry.opentofu.org/e-breuninger/netbox"; + }) + ]) diff --git a/deployment/run/tf-netbox-store-ips/variables.tf b/deployment/run/tf-netbox-store-ips/variables.tf new file mode 100644 index 00000000..61068fff --- /dev/null +++ b/deployment/run/tf-netbox-store-ips/variables.tf @@ -0,0 +1,9 @@ +variable "start_address" { + description = "Start of the IP range, e.g. 10.0.0.1/24." + type = string +} + +variable "end_address" { + description = "End of the IP range, e.g. 10.0.0.50/24." + type = string +} diff --git a/deployment/utils.nix b/deployment/utils.nix index 123669e4..992f465a 100644 --- a/deployment/utils.nix +++ b/deployment/utils.nix @@ -18,6 +18,19 @@ rec { ]; }).config; + evalOption = + name: opts: conf: + (lib.evalModules { + modules = [ + { + options = { + "${name}" = opts; + }; + config."${name}" = conf; + } + ]; + }).config."${name}"; + toBash = v: lib.replaceStrings [ "\"" ] [ "\\\"" ] ( diff --git a/npins/sources.json b/npins/sources.json index 3f7e07a9..f9806543 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -206,6 +206,22 @@ "url": "https://github.com/SaumonNet/proxmox-nixos/archive/ce8768f43b4374287cd8b88d8fa9c0061e749d9a.tar.gz", "hash": "116zplxh64wxbq81wsfkmmssjs1l228kvhxfi9d434xd54k6vr35" }, + "terraform-provider-netbox": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "e-breuninger", + "repo": "terraform-provider-netbox" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v5.0.0", + "revision": "40184568f1e7a626b44d5887d7d298866204733d", + "url": "https://api.github.com/repos/e-breuninger/terraform-provider-netbox/tarball/v5.0.0", + "hash": "1acpzxhvl6mz8fl4smcgy0l2wxkqrwywl13lwfj114svqsvq49l8" + }, "terraform-provider-proxmox": { "type": "Git", "repository": {