Unflakify deployment tests (#449)

This PR builds on top of #447 and #448. Since these might be rejected, there will be some changes needed for this PR as well. Let's see how the discussions go in #447.

In the meantime, @fricklerhandwerk, would you mind (in)validating the core idea of this PR? You only need to look at 7cf43c4041, really.

Reviewed-on: Fediversity/Fediversity#449
Reviewed-by: kiara Grouwstra <kiara@procolix.eu>
Reviewed-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Co-authored-by: Nicolas “Niols” Jeannerod <nicolas.jeannerod@moduscreate.com>
Co-committed-by: Nicolas “Niols” Jeannerod <nicolas.jeannerod@moduscreate.com>

deployment/check/basic/constants.nix

@@ -0,0 +1,8 @@
+{
+  targetMachines = [
+    "hello"
+    "cowsay"
+  ];
+  pathToRoot = ../../..;
+  pathFromRoot = ./.;
+}

deployment/check/basic/default.nix

@@ -0,0 +1,14 @@
+{
+  runNixOSTest,
+  inputs,
+  sources,
+}:
+
+runNixOSTest {
+  imports = [
+    ../common/nixosTest.nix
+    ./nixosTest.nix
+  ];
+  _module.args = { inherit inputs sources; };
+  inherit (import ./constants.nix) targetMachines pathToRoot pathFromRoot;
+}

deployment/check/basic/deployment.nix

@@ -0,0 +1,36 @@
+{
+  inputs,
+  sources,
+  lib,
+  providers,
+  ...
+}:
+
+let
+  inherit (import ./constants.nix) targetMachines pathToRoot pathFromRoot;
+in
+
+{
+  providers = {
+    inherit (inputs.nixops4.modules.nixops4Provider) local;
+  };
+
+  resources = lib.genAttrs targetMachines (nodeName: {
+    type = providers.local.exec;
+
+    imports = [
+      inputs.nixops4-nixos.modules.nixops4Resource.nixos
+      ../common/targetResource.nix
+    ];
+
+    _module.args = { inherit inputs sources; };
+
+    inherit nodeName pathToRoot pathFromRoot;
+
+    nixos.module =
+      { pkgs, ... }:
+      {
+        environment.systemPackages = [ pkgs.${nodeName} ];
+      };
+  });
+}

deployment/check/basic/flake-part.nix

@@ -1,57 +0,0 @@
-{
-  self,
-  inputs,
-  lib,
-  sources,
-  ...
-}:
-
-let
-  inherit (lib) genAttrs;
-
-  targetMachines = [
-    "hello"
-    "cowsay"
-  ];
-  pathToRoot = /. + (builtins.unsafeDiscardStringContext self);
-  pathFromRoot = ./.;
-
-in
-{
-  _class = "flake";
-
-  perSystem =
-    { pkgs, ... }:
-    {
-      checks.deployment-basic = pkgs.testers.runNixOSTest {
-        imports = [
-          ../common/nixosTest.nix
-          ./nixosTest.nix
-        ];
-        _module.args = { inherit inputs sources; };
-        inherit targetMachines pathToRoot pathFromRoot;
-      };
-    };
-
-  nixops4Deployments.check-deployment-basic =
-    { providers, ... }:
-    {
-      providers = {
-        inherit (inputs.nixops4.modules.nixops4Provider) local;
-      };
-      resources = genAttrs targetMachines (nodeName: {
-        type = providers.local.exec;
-        imports = [
-          inputs.nixops4-nixos.modules.nixops4Resource.nixos
-          ../common/targetResource.nix
-        ];
-        _module.args = { inherit inputs sources; };
-        inherit nodeName pathToRoot pathFromRoot;
-        nixos.module =
-          { pkgs, ... }:
-          {
-            environment.systemPackages = [ pkgs.${nodeName} ];
-          };
-      });
-    };
-}

deployment/check/basic/flake-under-test.nix

@@ -0,0 +1,22 @@
+{
+  inputs = {
+    nixops4.follows = "nixops4-nixos/nixops4";
+    nixops4-nixos.url = "github:nixops4/nixops4-nixos";
+  };
+
+  outputs =
+    inputs:
+    import ./mkFlake.nix inputs (
+      { inputs, sources, ... }:
+      {
+        imports = [
+          inputs.nixops4.modules.flake.default
+        ];
+
+        nixops4Deployments.check-deployment-basic = {
+          imports = [ ./deployment/check/basic/deployment.nix ];
+          _module.args = { inherit inputs sources; };
+        };
+      }
+    );
+}

deployment/check/cli/constants.nix

@@ -0,0 +1,11 @@
+{
+  targetMachines = [
+    "garage"
+    "mastodon"
+    "peertube"
+    "pixelfed"
+  ];
+  pathToRoot = ../../..;
+  pathFromRoot = ./.;
+  enableAcme = true;
+}

deployment/check/cli/default.nix

@@ -0,0 +1,19 @@
+{
+  runNixOSTest,
+  inputs,
+  sources,
+}:
+
+runNixOSTest {
+  imports = [
+    ../common/nixosTest.nix
+    ./nixosTest.nix
+  ];
+  _module.args = { inherit inputs sources; };
+  inherit (import ./constants.nix)
+    targetMachines
+    pathToRoot
+    pathFromRoot
+    enableAcme
+    ;
+}

deployment/check/cli/deployments.nix

@@ -0,0 +1,59 @@
+{
+  inputs,
+  sources,
+  lib,
+}:
+
+let
+  inherit (builtins) fromJSON readFile listToAttrs;
+  inherit (import ./constants.nix)
+    targetMachines
+    pathToRoot
+    pathFromRoot
+    enableAcme
+    ;
+
+  makeTargetResource = nodeName: {
+    imports = [ ../common/targetResource.nix ];
+    _module.args = { inherit inputs sources; };
+    inherit
+      nodeName
+      pathToRoot
+      pathFromRoot
+      enableAcme
+      ;
+  };
+
+  ## The deployment function - what we are here to test!
+  ##
+  ## TODO: Modularise `deployment/default.nix` to get rid of the nested
+  ## function calls.
+  makeTestDeployment =
+    args:
+    (import ../..)
+      {
+        inherit lib;
+        inherit (inputs) nixops4 nixops4-nixos;
+        fediversity = import ../../../services/fediversity;
+      }
+      (listToAttrs (
+        map (nodeName: {
+          name = "${nodeName}ConfigurationResource";
+          value = makeTargetResource nodeName;
+        }) targetMachines
+      ))
+      (fromJSON (readFile ../../configuration.sample.json) // args);
+
+in
+{
+  check-deployment-cli-nothing = makeTestDeployment { };
+
+  check-deployment-cli-mastodon-pixelfed = makeTestDeployment {
+    mastodon.enable = true;
+    pixelfed.enable = true;
+  };
+
+  check-deployment-cli-peertube = makeTestDeployment {
+    peertube.enable = true;
+  };
+}

deployment/check/cli/flake-part.nix

@@ -1,90 +0,0 @@
-{
-  self,
-  inputs,
-  lib,
-  sources,
-  ...
-}:
-
-let
-  inherit (builtins) fromJSON readFile listToAttrs;
-
-  targetMachines = [
-    "garage"
-    "mastodon"
-    "peertube"
-    "pixelfed"
-  ];
-  pathToRoot = /. + (builtins.unsafeDiscardStringContext self);
-  pathFromRoot = ./.;
-  enableAcme = true;
-
-in
-{
-  _class = "flake";
-
-  perSystem =
-    { pkgs, ... }:
-    {
-      checks.deployment-cli = pkgs.testers.runNixOSTest {
-        imports = [
-          ../common/nixosTest.nix
-          ./nixosTest.nix
-        ];
-        _module.args = { inherit inputs sources; };
-        inherit
-          targetMachines
-          pathToRoot
-          pathFromRoot
-          enableAcme
-          ;
-      };
-    };
-
-  nixops4Deployments =
-    let
-      makeTargetResource = nodeName: {
-        imports = [ ../common/targetResource.nix ];
-        _module.args = { inherit inputs sources; };
-        inherit
-          nodeName
-          pathToRoot
-          pathFromRoot
-          enableAcme
-          ;
-      };
-
-      ## The deployment function - what we are here to test!
-      ##
-      ## TODO: Modularise `deployment/default.nix` to get rid of the nested
-      ## function calls.
-      makeTestDeployment =
-        args:
-        (import ../..)
-          {
-            inherit lib;
-            inherit (inputs) nixops4 nixops4-nixos;
-            fediversity = import ../../../services/fediversity;
-          }
-          (listToAttrs (
-            map (nodeName: {
-              name = "${nodeName}ConfigurationResource";
-              value = makeTargetResource nodeName;
-            }) targetMachines
-          ))
-          (fromJSON (readFile ../../configuration.sample.json) // args);
-
-    in
-    {
-      check-deployment-cli-nothing = makeTestDeployment { };
-
-      check-deployment-cli-mastodon-pixelfed = makeTestDeployment {
-        mastodon.enable = true;
-        pixelfed.enable = true;
-      };
-
-      check-deployment-cli-peertube = makeTestDeployment {
-        peertube.enable = true;
-      };
-    };
-}

deployment/check/cli/flake-under-test.nix

@@ -0,0 +1,26 @@
+{
+  inputs = {
+    nixops4.follows = "nixops4-nixos/nixops4";
+    nixops4-nixos.url = "github:nixops4/nixops4-nixos";
+  };
+
+  outputs =
+    inputs:
+    import ./mkFlake.nix inputs (
+      {
+        inputs,
+        sources,
+        lib,
+        ...
+      }:
+      {
+        imports = [
+          inputs.nixops4.modules.flake.default
+        ];
+
+        nixops4Deployments = import ./deployment/check/cli/deployments.nix {
+          inherit inputs sources lib;
+        };
+      }
+    );
+}

deployment/check/common/nixosTest.nix

@@ -134,7 +134,13 @@ in
 
       ## NOTE: This is super slow. It could probably be optimised in Nix, for
       ## instance by allowing to grab things directly from the host's store.
-      with subtest("Override the lock"):
+      ##
+      ## NOTE: We use the repository as-is (cf `src` above), overriding only
+      ## `flake.nix` by our `flake-under-test.nix`. We also override the flake
+      ## lock file to use locally available inputs, as we cannot download them.
+      ##
+      with subtest("Override the flake and its lock"):
+        deployer.succeed("cp ${config.pathFromRoot}/flake-under-test.nix flake.nix")
         deployer.succeed("""
           nix flake lock --extra-experimental-features 'flakes nix-command' \
             --offline -v \

deployment/check/panel/constants.nix

@@ -0,0 +1,11 @@
+{
+  targetMachines = [
+    "garage"
+    "mastodon"
+    "peertube"
+    "pixelfed"
+  ];
+  pathToRoot = ../../..;
+  pathFromRoot = ./.;
+  enableAcme = true;
+}

deployment/check/panel/default.nix

@@ -0,0 +1,19 @@
+{
+  runNixOSTest,
+  inputs,
+  sources,
+}:
+
+runNixOSTest {
+  imports = [
+    ../common/nixosTest.nix
+    ./nixosTest.nix
+  ];
+  _module.args = { inherit inputs sources; };
+  inherit (import ./constants.nix)
+    targetMachines
+    pathToRoot
+    pathFromRoot
+    enableAcme
+    ;
+}

deployment/check/panel/deployment.nix

@@ -0,0 +1,58 @@
+{
+  inputs,
+  sources,
+  lib,
+}:
+
+let
+  inherit (builtins) fromJSON listToAttrs;
+  inherit (import ./constants.nix)
+    targetMachines
+    pathToRoot
+    pathFromRoot
+    enableAcme
+    ;
+
+  makeTargetResource = nodeName: {
+    imports = [ ../common/targetResource.nix ];
+    _module.args = { inherit inputs sources; };
+    inherit
+      nodeName
+      pathToRoot
+      pathFromRoot
+      enableAcme
+      ;
+  };
+
+  ## The deployment function - what we are here to test!
+  ##
+  ## TODO: Modularise `deployment/default.nix` to get rid of the nested
+  ## function calls.
+  makeTestDeployment =
+    args:
+    (import ../..)
+      {
+        inherit lib;
+        inherit (inputs) nixops4 nixops4-nixos;
+        fediversity = import ../../../services/fediversity;
+      }
+      (listToAttrs (
+        map (nodeName: {
+          name = "${nodeName}ConfigurationResource";
+          value = makeTargetResource nodeName;
+        }) targetMachines
+      ))
+      args;
+
+in
+makeTestDeployment (
+  fromJSON (
+    let
+      env = builtins.getEnv "DEPLOYMENT";
+    in
+    if env == "" then
+      throw "The DEPLOYMENT environment needs to be set. You do not want to use this deployment unless in the `deployment-panel` NixOS test."
+    else
+      env
+  )
+)

deployment/check/panel/flake-part.nix

@@ -1,94 +0,0 @@
-{
-  self,
-  inputs,
-  lib,
-  sources,
-  ...
-}:
-
-let
-  inherit (builtins)
-    fromJSON
-    listToAttrs
-    ;
-
-  targetMachines = [
-    "garage"
-    "mastodon"
-    "peertube"
-    "pixelfed"
-  ];
-  pathToRoot = /. + (builtins.unsafeDiscardStringContext self);
-  pathFromRoot = ./.;
-  enableAcme = true;
-
-in
-{
-  _class = "flake";
-
-  perSystem =
-    { pkgs, ... }:
-    {
-      checks.deployment-panel = pkgs.testers.runNixOSTest {
-        imports = [
-          ../common/nixosTest.nix
-          ./nixosTest.nix
-        ];
-        _module.args = { inherit inputs sources; };
-        inherit
-          targetMachines
-          pathToRoot
-          pathFromRoot
-          enableAcme
-          ;
-      };
-    };
-
-  nixops4Deployments =
-    let
-      makeTargetResource = nodeName: {
-        imports = [ ../common/targetResource.nix ];
-        _module.args = { inherit inputs sources; };
-        inherit
-          nodeName
-          pathToRoot
-          pathFromRoot
-          enableAcme
-          ;
-      };
-
-      ## The deployment function - what we are here to test!
-      ##
-      ## TODO: Modularise `deployment/default.nix` to get rid of the nested
-      ## function calls.
-      makeTestDeployment =
-        args:
-        (import ../..)
-          {
-            inherit lib;
-            inherit (inputs) nixops4 nixops4-nixos;
-            fediversity = import ../../../services/fediversity;
-          }
-          (listToAttrs (
-            map (nodeName: {
-              name = "${nodeName}ConfigurationResource";
-              value = makeTargetResource nodeName;
-            }) targetMachines
-          ))
-          args;
-
-    in
-    {
-      check-deployment-panel = makeTestDeployment (
-        fromJSON (
-          let
-            env = builtins.getEnv "DEPLOYMENT";
-          in
-          if env == "" then
-            throw "The DEPLOYMENT environment needs to be set. You do not want to use this deployment unless in the `deployment-panel` NixOS test."
-          else
-            env
-        )
-      );
-    };
-}

deployment/check/panel/flake-under-test.nix

@@ -0,0 +1,26 @@
+{
+  inputs = {
+    nixops4.follows = "nixops4-nixos/nixops4";
+    nixops4-nixos.url = "github:nixops4/nixops4-nixos";
+  };
+
+  outputs =
+    inputs:
+    import ./mkFlake.nix inputs (
+      {
+        inputs,
+        sources,
+        lib,
+        ...
+      }:
+      {
+        imports = [
+          inputs.nixops4.modules.flake.default
+        ];
+
+        nixops4Deployments.check-deployment-panel = import ./deployment/check/panel/deployment.nix {
+          inherit inputs sources lib;
+        };
+      }
+    );
+}

deployment/flake-part.nix

@@ -1,9 +1,26 @@
+{ inputs, sources, ... }:
+
 {
   _class = "flake";
 
-  imports = [
+  perSystem =
-    ./check/basic/flake-part.nix
+    { pkgs, ... }:
-    ./check/cli/flake-part.nix
+    {
-    ./check/panel/flake-part.nix
+      checks = {
-  ];
+        deployment-basic = import ./check/basic {
+          inherit (pkgs.testers) runNixOSTest;
+          inherit inputs sources;
+        };
+
+        deployment-cli = import ./check/cli {
+          inherit (pkgs.testers) runNixOSTest;
+          inherit inputs sources;
+        };
+
+        deployment-panel = import ./check/panel {
+          inherit (pkgs.testers) runNixOSTest;
+          inherit inputs sources;
+        };
+      };
+    };
 }