init

README.md

@@ -0,0 +1,7 @@
+# vars
+
+This repo is for testing https://github.com/NixOS/nixpkgs/pull/370444 without patching your nixpkgs or running clan.
+
+DavHau also wrote a blogpost about the more advanced version from clan.lol: https://clan.lol/blog/vars/
+
+TODO: add more README

backends/on-machine.nix

@@ -0,0 +1,140 @@
+# we use this vars backend as an example backend.
+# this generates a script which creates the values at the expected path.
+# this script has to be run manually (I guess after updating the system) to generate the required vars.
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.vars.settings.on-machine;
+  sortedGenerators =
+    (lib.toposort (a: b: builtins.elem a.name b.dependencies) (lib.attrValues config.vars.generators))
+    .result;
+
+  promptCmd = {
+    hidden = "read -sr prompt_value";
+    line = "read -r prompt_value";
+    multiline = ''
+      echo 'press control-d to finish'
+      prompt_value=$(cat)
+    '';
+  };
+  generate-vars = pkgs.writeShellApplication {
+    name = "generate-vars";
+    text = ''
+      set -efuo pipefail
+
+      PATH=${lib.makeBinPath [ pkgs.coreutils ]}
+
+      # make the output directory overridable
+      OUT_DIR=''${OUT_DIR:-${cfg.fileLocation}}
+
+      # check if all files are present or all files are missing
+      # if not, they are in an inconsistent state and we bail out
+      ${lib.concatMapStringsSep "\n" (gen: ''
+        all_files_missing=true
+        all_files_present=true
+        ${lib.concatMapStringsSep "\n" (file: ''
+          if test -e ${lib.escapeShellArg file.path} ; then
+            all_files_missing=false
+          else
+            all_files_present=false
+          fi
+        '') (lib.attrValues gen.files)}
+
+        if [ $all_files_missing = false ] && [ $all_files_present = false ] ; then
+          echo "Inconsistent state for generator: {gen.name}"
+          exit 1
+        fi
+        if [ $all_files_present = true ] ; then
+          echo "All secrets for ${gen.name} are present"
+        elif [ $all_files_missing = true ] ; then
+
+          # prompts
+          prompts=$(mktemp -d)
+          trap 'rm -rf $prompts' EXIT
+          export prompts
+          mkdir -p "$prompts"
+          ${lib.concatMapStringsSep "\n" (prompt: ''
+            echo ${lib.escapeShellArg prompt.description}
+            ${promptCmd.${prompt.type}}
+            echo -n "$prompt_value" > "$prompts"/${prompt.name}
+          '') (lib.attrValues gen.prompts)}
+          echo "Generating vars for ${gen.name}"
+
+          # dependencies
+          in=$(mktemp -d)
+          trap 'rm -rf $in' EXIT
+          export in
+          mkdir -p "$in"
+          ${lib.concatMapStringsSep "\n" (input: ''
+            mkdir -p "$in"/${input}
+            ${lib.concatMapStringsSep "\n" (file: ''
+              cp "$OUT_DIR"/${
+                if file.secret then "secret" else "public"
+              }/${input}/${file.name} "$in"/${input}/${file.name}
+            '') (lib.attrValues config.vars.generators.${input}.files)}
+          '') gen.dependencies}
+
+          # outputs
+          out=$(mktemp -d)
+          trap 'rm -rf $out' EXIT
+          export out
+          mkdir -p "$out"
+
+          (
+            # prepare PATH
+            unset PATH
+            ${lib.optionalString (gen.runtimeInputs != [ ]) ''
+              PATH=${lib.makeBinPath gen.runtimeInputs}
+              export PATH
+            ''}
+
+            # actually run the generator
+            ${gen.script}
+          )
+
+          # check if all files got generated
+          ${lib.concatMapStringsSep "\n" (file: ''
+            if ! test -e "$out"/${file.name} ; then
+              echo 'generator ${gen.name} failed to generate ${file.name}'
+              exit 1
+            fi
+          '') (lib.attrValues gen.files)}
+
+          # move the files to the correct location
+          ${lib.concatMapStringsSep "\n" (file: ''
+            OUT_FILE="$OUT_DIR"/${if file.secret then "secret" else "public"}/${file.generator}/${file.name}
+            mkdir -p "$(dirname "$OUT_FILE")"
+            mv "$out"/${file.name} "$OUT_FILE"
+          '') (lib.attrValues gen.files)}
+          rm -rf "$out"
+        fi
+      '') sortedGenerators}
+    '';
+  };
+in
+{
+  options.vars.settings.on-machine = {
+    enable = lib.mkEnableOption "Enable on-machine vars backend";
+    fileLocation = lib.mkOption {
+      type = lib.types.str;
+      default = "/etc/vars";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    vars.settings.fileModule = file: {
+      path =
+        if file.config.secret then
+          "${cfg.fileLocation}/secret/${file.config.generator}/${file.config.name}"
+        else
+          "${cfg.fileLocation}/public/${file.config.generator}/${file.config.name}";
+    };
+    environment.systemPackages = [
+      generate-vars
+    ];
+    system.build.generate-vars = generate-vars;
+  };
+}

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1743583204,
+        "narHash": "sha256-F7n4+KOIfWrwoQjXrL2wD9RhFYLs2/GGe/MQY1sSdlE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2c8d3f48d33929642c1c12cd243df4cc7d2ce434",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,32 @@
+{
+  description = "testing vars without depending on clan";
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+  outputs = inputs:
+    let
+      lib = inputs.nixpkgs.lib;
+      supportedSystems = [
+        "x86_64-linux"
+        "i686-linux"
+        "aarch64-linux"
+        "riscv64-linux"
+      ];
+      forAllSystems = lib.genAttrs supportedSystems;
+    in {
+      nixosModules.default = { imports = [ ./options.nix ]; };
+      nixosModules.backend-on-machine = { imports = [ ./backends/on-machine.nix ]; };
+      # TODO fix tests
+      checks = forAllSystems (system: let
+        tests = {
+          testing = inputs.nixpkgs.lib.nixos.runTest {
+            hostPkgs = inputs.nixpkgs.legacyPackages.${system};
+            imports = [
+              ./options.nix
+              ./testing.nix
+            ];
+          };
+        };
+      in tests);
+    };
+}

options.nix

@@ -0,0 +1,185 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+{
+  options.vars = {
+    settings = {
+      fileModule = lib.mkOption {
+        type = lib.types.deferredModule;
+        internal = true;
+        description = ''
+          A module to be imported in every vars.files.<name> submodule.
+          Used by backends to define the `path` attribute.
+
+          Takes the file as an argument and returns maybe an attrset which should at least contain the `path` attribute.
+          Can be used to set other file attributes as well, like `value`.
+        '';
+        default = { };
+      };
+    };
+    generators = lib.mkOption {
+      description = ''
+        A set of generators that can be used to generate files.
+        Generators are scripts that produce files based on the values of other generators and user input.
+        Each generator is expected to produce a set of files under a directory.
+      '';
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (generator: {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.strMatching "[a-zA-Z0-9:_\\.-]*";
+              description = ''
+                The name of the generator.
+                This name will be used to refer to the generator in other generators.
+              '';
+              readOnly = true;
+              default = generator.config._module.args.name;
+              defaultText = "Name of the generator";
+            };
+
+            dependencies = lib.mkOption {
+              description = ''
+                A list of other generators that this generator depends on.
+                The output values of these generators will be available to the generator script as files.
+                For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1.
+              '';
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+            };
+            files = lib.mkOption {
+              description = ''
+                A set of files to generate.
+                The generator 'script' is expected to produce exactly these files under $out.
+              '';
+              defaultText = "attrs of files";
+              type = lib.types.attrsOf (
+                lib.types.submodule (file: {
+                  imports = [
+                    config.vars.settings.fileModule
+                  ];
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.strMatching "[a-zA-Z0-9:_\\.-]*";
+                      description = ''
+                        name of the generated file
+                      '';
+                      readOnly = true;
+                      default = file.config._module.args.name;
+                      defaultText = "Name of the file";
+                    };
+                    generator = lib.mkOption {
+                      description = ''
+                        The generator that produces the file.
+                        This is the name of another generator.
+                      '';
+                      type = lib.types.strMatching "[a-zA-Z0-9:_\\.-]*";
+                      readOnly = true;
+                      internal = true;
+                      default = generator.config.name;
+                      defaultText = "Name of the generator";
+                    };
+                    deploy = lib.mkOption {
+                      description = ''
+                        Whether the file should be deployed to the target machine.
+
+                        Disable this if the generated file is only used as an input to other generators.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    secret = lib.mkOption {
+                      description = ''
+                        Whether the file should be treated as a secret.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    path = lib.mkOption {
+                      description = ''
+                        The path to the file containing the content of the generated value.
+                        This will be set automatically
+                      '';
+                      type = lib.types.nullOr lib.types.str;
+                      default = null;
+                    };
+                  };
+                })
+              );
+            };
+            prompts = lib.mkOption {
+              description = ''
+                A set of prompts to ask the user for values.
+                Prompts are available to the generator script as files.
+                For example, a prompt named 'prompt1' will be available via $prompts/prompt1
+              '';
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (prompt: {
+                  options = {
+                    name = lib.mkOption {
+                      description = ''
+                        The name of the prompt.
+                        This name will be used to refer to the prompt in the generator script.
+                      '';
+                      type = lib.types.strMatching "[a-zA-Z0-9:_\\.-]*";
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    description = lib.mkOption {
+                      description = ''
+                        The description of the prompted value
+                      '';
+                      type = lib.types.str;
+                      example = "SSH private key";
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    type = lib.mkOption {
+                      description = ''
+                        The input type of the prompt.
+                        The following types are available:
+                          - hidden: A hidden text (e.g. password)
+                          - line: A single line of text
+                          - multiline: A multiline text
+                      '';
+                      type = lib.types.enum [
+                        "hidden"
+                        "line"
+                        "multiline"
+                      ];
+                      default = "line";
+                    };
+                  };
+                })
+              );
+            };
+            runtimeInputs = lib.mkOption {
+              description = ''
+                A list of packages that the generator script requires.
+                These packages will be available in the PATH when the script is run.
+              '';
+              type = lib.types.listOf lib.types.package;
+              default = [ pkgs.coreutils ];
+            };
+            script = lib.mkOption {
+              description = ''
+                The script to run to generate the files.
+                The script will be run with the following environment variables:
+                  - $in: The directory containing the output values of all declared dependencies
+                  - $out: The output directory to put the generated files
+                  - $prompts: The directory containing the prompted values as files
+                The script should produce the files specified in the 'files' attribute under $out.
+              '';
+              type = lib.types.either lib.types.str lib.types.path;
+              default = "";
+            };
+          };
+        })
+      );
+    };
+  };
+}

testing.nix

@@ -0,0 +1,117 @@
+{ lib, ... }:
+
+{
+  name = "vars";
+  meta.maintainers = with lib.maintainers; [ lassulus ];
+
+  nodes.machine =
+    { ... }:
+    {
+      imports = [
+        ./options.nix
+        ./backends/on-machine.nix
+      ];
+      vars.settings.on-machine.enable = true;
+      vars.generators = {
+        simple = {
+          files.simple = { };
+          script = ''
+            echo simple > "$out"/simple
+          '';
+        };
+
+        a = {
+          files.a = { };
+          script = ''
+            echo a > "$out"/a
+          '';
+        };
+        b = {
+          dependencies = [ "a" ];
+          files.b = { };
+          script = ''
+            cat "$in"/a/a > "$out"/b
+            echo b >> "$out"/b
+          '';
+        };
+
+        prompts = {
+          files.prompt_line = { };
+          files.prompt_hidden = { };
+          files.prompt_multiline = { };
+          prompts.line = {
+            description = ''
+              a simple line prompt
+            '';
+          };
+          prompts.hidden = {
+            type = "hidden";
+            description = ''
+              a prompt that doesn't show the input
+            '';
+          };
+          prompts.aamulti = {
+            type = "multiline";
+            description = ''
+              a prompt with multiple lines
+            '';
+          };
+          script = ''
+            cp "$prompts"/line "$out"/prompt_line
+            cp "$prompts"/hidden "$out"/prompt_hidden
+            cp "$prompts"/aamulti "$out"/prompt_multiline
+          '';
+        };
+      };
+    };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      import subprocess
+      from pathlib import Path
+
+      process = subprocess.Popen(
+        ["${nodes.machine.config.system.build.generate-vars}/bin/generate-vars"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        text=True,
+        env={
+          "OUT_DIR": "./vars",
+        },
+      )
+
+      # Function to check for expected outputs and send corresponding texts
+
+      def interact_with_process(process, interactions):
+          while interactions:
+              output = process.stdout.readline()
+              if output:
+                  print(output.strip())  # Print the output for debugging
+                  for expected_output, text_to_send in interactions:
+                      if expected_output in output:
+                          print("sending", text_to_send)
+                          process.stdin.write(text_to_send + '\n')
+                          process.stdin.flush()
+                          interactions.remove((expected_output, text_to_send))
+                          break
+
+      interactions = [
+          ("a simple line prompt", "simple prompt content"),
+          ("a prompt that doesn't show the input", "hidden prompt content"),
+          ("press control-d to finish", f"multi line content1\nmulti line content2\n\n{chr(4)}\n"),
+          ("another prompt after EOF", "another prompt content"),
+      ]
+
+      # Interact with the process
+      interact_with_process(process, interactions)
+
+      # Wait for the process to complete
+      process.wait()
+
+      vars_folder = Path("vars")
+      print(list(vars_folder.glob("*")))
+      assert((vars_folder / "secret" /  "a" / "a").exists())
+
+    '';
+}