diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 30f8d61a2c705..8759a0330a45a 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -22571,6 +22571,13 @@ githubId = 11898437; name = "Florian Ströger"; }; + prescientmoon = { + email = "nix@moonythm.dev"; + matrix = "@prescientmoon:moonythm.dev"; + github = "prescientmoon"; + githubId = 39400800; + name = "prescientmoon"; + }; presto8 = { name = "Preston Hunt"; email = "me@prestonhunt.com"; diff --git a/nixos/modules/security/secrets/default.nix b/nixos/modules/security/secrets/default.nix new file mode 100644 index 0000000000000..850754eb59d9d --- /dev/null +++ b/nixos/modules/security/secrets/default.nix @@ -0,0 +1,338 @@ +{ lib, config, ... }: +with lib.types; +let + # A name containing only "safe" characters that we allow passing to the + # backends. + # + # Note that we often default options of type `safeName` to Nix attribute + # names. For example, + # ``` + # secrets.store."foo bar" = {} + # ``` + # will implicitly set `secrets.store."foo bar".name = "foo bar"`. This is + # bad, because the error will get a confusing error coming from the wrong + # place. In the future, it might be worth creating a modified version of + # `attrsOf` that does not have this issue, although the added complexity is + # not worth it right now. + safeName = + of: + addCheck str ( + s: + if lib.strings.match "^[a-zA-Z0-9:_\\.-]+$" s == null then + throw "Name '${toString s}' is not a valid ${of} name. Currently, only alphanumeric characters, dashes, underscores, and dots are allowed." + else + true + ); + cfg = config.secrets; + + deferredPackage = + description: + lib.mkOption { + inherit description; + type = functionTo pathInStore; + example = pkgs: pkgs.writeShellScript "example" "echo 'Hi!'"; + }; + + nullableDeferredPackage = + description: + lib.mkOption { + inherit description; + type = nullOr (functionTo pathInStore); + example = pkgs: pkgs.writeShellScript "example" "echo 'Hi!'"; + default = null; + }; + + storeBackendModule = submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + description = "The name of the backend."; + type = str; + default = name; + }; + + get = nullableDeferredPackage '' + Given $1=gen_name and $2=file_name, the script retrieves the + respective secret to $out. + ''; + + set = deferredPackage '' + Given $1=gen_name and $2=file_name, the script retrieves the + respective secret from $in and stores it in the appropriate location. + ''; + + delete = nullableDeferredPackage '' + Given $1=gen_name and $2=file_name, the script deletes the respective + secret if it does exist. + ''; + + list = deferredPackage '' + A script that lists all files managed by this backend. Should output + space-separated or newline-separated pairs of: secret_name + file_name. + + If the backend supports multiple hosts, then this command should only + list the secrets owned by the current host. In particular, files + included in this command's output will be deleted by the + collect-garbage command, unless they appear in the user's secret + configuration. + + This script must not perform side effects. + ''; + + fixup = nullableDeferredPackage '' + This script will be run on every invocation of the CLI's "generate" + command. Given $1=file_list in the same format used by `list`, the + script performs any necessary updates to the secrets' files (e.g. + rekeying encrypted secrets). This script can perform side effects, + but must be idempotent. + ''; + + deploy.remote = nullableDeferredPackage '' + Deploys every available file to the given machine. The list of files + to deploy is provided as $1 in the same format used by `list`. Any + additional information required by the deploy script can be provided + by the user through environment variables. + ''; + + deploy.local = nullableDeferredPackage '' + Deploys every available file to the machine with system root mounted + at $1=system_root. The list of files to deploy is provided as $2 in + the same format used by `list`. This is useful for fresh installs + from environments live live CDs, where the target system is not yet + up and running (even if nixos-install has successfully completed). + ''; + + fileModule = lib.mkOption { + type = deferredModule; + internal = true; + default = { }; + description = '' + A module to be imported in every + secrets.store..files. submodule. Used by backends to + define the `path` attribute. The module will have the following + additional arguments passed to it: + - `secret`, containing the secret the file belongs to + - `backend`, containing the backend associated with said secret + ''; + }; + }; + } + ); + + fileModule = + { name, backend, ... }: + { + imports = [ backend.fileModule ]; + options = { + name = lib.mkOption { + description = "name of the file"; + type = safeName "file"; + default = name; + defaultText = "Name of the file"; + }; + + path = lib.mkOption { + description = "Path to the file; usually set by the backend"; + type = path; + }; + + deploy = lib.mkOption { + description = '' + Files with this flag will be included in the file list passed to + the `deploy` script. + ''; + type = bool; + default = true; + }; + }; + }; + + secretModule = submodule ( + { name, config, ... }: + let + backend = cfg.backends.store.${config.backend}; + in + { + options = { + name = lib.mkOption { + description = '' + The name of the secret. This name will be used to refer to the + the secrets from other generators. + ''; + type = safeName "secret"; + default = name; + defaultText = "Attribute name of the secret"; + }; + + backend = lib.mkOption { + type = str; + description = "The backend responsible for handling this secret."; + default = cfg.backends.defaults.store; + }; + + prompts = lib.mkOption { + description = '' + A set of prompts the generator will have at its disposal. + ''; + default = { }; + type = attrsOf promptModule; + }; + + dependencies = lib.mkOption { + description = '' + A list of other secrets this generator should be able to read the + output(s) of. + ''; + type = listOf (safeName "secret"); + default = [ ]; + }; + + files = lib.mkOption { + description = '' + A set of files to store. The generator 'script' is expected to + produce exactly these files under $out. + ''; + default = { }; + type = attrsOf (submoduleWith { + modules = [ fileModule ]; + specialArgs = { + inherit backend; + secret = config; + }; + }); + }; + + generate = nullableDeferredPackage '' + The script to run to generate the files. The script will be run with + the following environment variables: + - $in: The directory containing the output values of all declared + dependencies + - $out: The output directory to put the generated files + - $prompts: The directory containing the prompted values as files + The script should produce the files specified in the 'files' attribute + under $out. + ''; + }; + } + ); + + promptBackendModule = submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + description = "The name of the backend."; + type = str; + default = name; + }; + + ask = deferredPackage '' + Given $1=secret_name, $2=prompt_name, $3=prompt_type, + $4=prompt_label, and optionally $5=prompt_description, the script + runs the prompt by the user, then saves respective value to $out. + + Do note that the given $2=prompt_name is not meant as a prompt label! + $4=prompt_label should be used for that purpose. Indeed, $1 and $2 + are only really useful for non-interactive usecases. + ''; + }; + } + ); + + promptModule = submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + description = "The prompt's name."; + type = safeName "prompt"; + default = name; + }; + + label = lib.mkOption { + description = "The label to attach to the prompt."; + type = str; + default = name; + }; + + description = lib.mkOption { + description = '' + An optional longer description of the prompted value. + ''; + type = nullOr str; + default = null; + example = "SSH private key"; + }; + + type = lib.mkOption { + description = '' + The input type of the prompt. + The following types are available: + - hidden: A hidden text (e.g. password) + - line: A single line of text + - multiline: A multiline text + ''; + type = enum [ + "hidden" + "line" + "multiline" + ]; + default = "line"; + }; + + backend = lib.mkOption { + type = str; + description = "The backend responsible for handling this prompt."; + default = cfg.backends.defaults.prompt; + }; + }; + } + ); +in +{ + options.secrets = { + store = lib.mkOption { + description = '' + A set of secrets that are each expected to store a set of files + under a directory. Generators can produce files using a script, + possibly referencing values produced by other generators and user + input. The secrets can also be manually imported from external files + using the CLI. + ''; + default = { }; + type = attrsOf secretModule; + }; + + backends.store = lib.mkOption { + description = '' + A set of backends that handle storing and retrieving secret files. + ''; + default = { }; + type = attrsOf storeBackendModule; + }; + + backends.defaults.store = lib.mkOption { + description = '' + The default backend to use for secrets that do not specify one. + ''; + type = str; + }; + + backends.prompt = lib.mkOption { + description = '' + A set of backends that handle retrieving user inputs. + ''; + default = { }; + type = attrsOf promptBackendModule; + }; + + backends.defaults.prompt = lib.mkOption { + description = '' + The default backend to use for prompts that do not specify one. + ''; + type = str; + }; + }; +} diff --git a/nixos/modules/security/secrets/example/README.md b/nixos/modules/security/secrets/example/README.md new file mode 100644 index 0000000000000..eca95c246d9d9 --- /dev/null +++ b/nixos/modules/security/secrets/example/README.md @@ -0,0 +1 @@ +This directory contains a bunch of examples of how one might use the `nixos-secrets` module. These are placed here such that the associated nixos-test(s) can reference the example backends via the `modulesPath` (as suggested by Lassulus). diff --git a/nixos/modules/security/secrets/example/basic-config/default.nix b/nixos/modules/security/secrets/example/basic-config/default.nix new file mode 100644 index 0000000000000..d142ed3dbd39a --- /dev/null +++ b/nixos/modules/security/secrets/example/basic-config/default.nix @@ -0,0 +1,75 @@ +{ lib, modulesPath, ... }: +{ + imports = [ + "${modulesPath}/security/secrets" + ../common/backend-plain.nix + ../common/backend-age.nix + ../common/backend-prompt-simple.nix + ]; + + secrets = { + backends.defaults.store = "plain"; + backends.defaults.prompt = "simple"; + + settings.store.age.publicKeys = [ + "age13ar5t7vvsssmckjhjtngy3p5y0v4k896ecjrxveql9ysu8gxhe9sdar3k3" # Host + "age195x33zrqzppjfnj2rjjlq3z8s64r5zlwe6rcywm9zu6agf449pmqdslyat" # Target + ]; + + # NOTE: do *not* do this with real keys!!! This will copy the keys to the + # world-readable Nix store, which is most probably not what you want! + settings.store.age.identity.host = toString ../common/key-host.txt; + settings.store.age.identity.target = toString ../common/key-target.txt; + + settings.store.age.ssh.target = "root@lapetus.overlay.moonythm.dev"; + settings.store.age.ssh.identity = "/home/moon/.ssh/id_ed25519"; + + store.user = { + # This prompt will default to the "simple" backend we chose above. + prompts.name = { + label = "Your name"; + description = "the person to address the greeting to"; + type = "multiline"; + }; + + files.greeting.deploy = false; + generate = + pkgs: + pkgs.writeScript "gen-user" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + echo "Hewwo $(cat "$prompts/name")!" > "$out/greeting" + ''; + }; + + store.derived = { + backend = "age"; + dependencies = [ "user" ]; + files.cow-greeting = { }; + generate = + pkgs: + pkgs.writeScript "gen-derived" '' + #!/bin/sh + export PATH="${ + lib.makeBinPath [ + pkgs.coreutils + pkgs.cowsay + ] + }" + cat $in/user/greeting | cowsay > $out/cow-greeting + ''; + }; + + store.derivedPlain = { + dependencies = [ "derived" ]; + files.cow-greeting-copy = { }; + generate = + pkgs: + pkgs.writeScript "gen-derived-plain" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + cat $in/derived/cow-greeting > $out/cow-greeting-copy + ''; + }; + }; +} diff --git a/nixos/modules/security/secrets/example/common/README.md b/nixos/modules/security/secrets/example/common/README.md new file mode 100644 index 0000000000000..9b0284842e5c9 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/README.md @@ -0,0 +1 @@ +This directory contains example backends for use in the various examples. Do note that these backends are _not_ production ready by any means. You should consider any code the backends are built upon as potentially unsafe. Indeed, their only purpose is demonstrating the `nixos-secrets` APIs. diff --git a/nixos/modules/security/secrets/example/common/backend-age.nix b/nixos/modules/security/secrets/example/common/backend-age.nix new file mode 100644 index 0000000000000..cb853852b7ff5 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-age.nix @@ -0,0 +1,189 @@ +# An example age-based secret backend written in Python. +{ + config, + lib, + ... +}: +let + cfg = config.secrets.settings.store.age; + + # This data will get encoded as JSON, and passed to every invocation of the + # backend's CLI. + ageNixConfig = { + generators = lib.pipe config.secrets.store [ + (lib.filterAttrs (_: secrets: secrets.backend == "age")) + (lib.mapAttrs' ( + _: secret: { + inherit (secret) name; + value = { + inherit (secret.age) publicKeys; + identity = { inherit (secret.age.identity) target host; }; + }; + } + )) + ]; + + inherit (cfg) + hostDirectory + targetDirectory + identity + publicKeys + ; + }; + + # We bake the configuration and required command into the script that calls + # the CLI. I'm not sure if doing it this way is better than overriding the + # Python writer directly. I guess this method shared the original Python + # script derivation, but that might not be meaningful for such a small script. + ageScript = + pkgs: command: + let + ageJSONConfig = pkgs.writeText "age.json" (builtins.toJSON ageNixConfig); + scriptSource = builtins.readFile ./backend-age.py; + raw = pkgs.writers.writePython3Bin "secrets-age-backend" { + flakeIgnore = [ + "W191" + "E501" + ]; + } scriptSource; + in + lib.getExe ( + pkgs.symlinkJoin { + name = "secrets-age-backend"; + paths = [ raw ]; + buildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/secrets-age-backend \ + --set PATH ${cfg.package pkgs}/bin \ + --add-flags "${ageJSONConfig} ${command}" + ''; + } + ); +in +{ + options.secrets.settings.store.age = { + package = lib.mkOption { + type = lib.types.functionTo lib.types.pathInStore; + default = pkgs: pkgs.age; + description = "The package to use for the 'age' CLI"; + }; + + hostDirectory = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets-ng-ng-age/host/${config.networking.hostName}"; + description = '' + The directory where the age backend will store encrypted secrets on the + host machine. + ''; + }; + + targetDirectory = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets-ng-ng-age/target"; + description = '' + The directory where the age backend will store encrypted secrets on the + target machine. + ''; + }; + + publicKeys = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Age public keys to encrypt to"; + }; + + identity.target = lib.mkOption { + type = lib.types.str; + description = '' + Path to the age private key file for decryption on the target machine + ''; + + example = "/var/lib/nixos-secrets/age.key"; + }; + + identity.host = lib.mkOption { + type = lib.types.str; + description = '' + Path to the age private key file for decryption on the host machine + ''; + }; + + ssh.identity = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + The private key to use when deploying over SSH. + ''; + }; + + ssh.target = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "eve@example.com"; + description = '' + The target to deploy files over SSH to. + ''; + }; + }; + + options.secrets.store = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options.age = { + publicKeys = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = cfg.publicKeys; + description = "Age public keys to encrypt to"; + }; + + identity.target = lib.mkOption { + default = cfg.identity.target; + type = lib.types.str; + description = '' + Path to the age private key file for decryption on the target + machine + ''; + }; + + identity.host = lib.mkOption { + default = cfg.identity.host; + type = lib.types.str; + description = '' + Path to the age private key file for decryption on the host machine + ''; + }; + }; + } + ); + }; + + config.secrets.backends.store.age = { + get = pkgs: ageScript pkgs "get"; + set = pkgs: ageScript pkgs "set"; + list = pkgs: ageScript pkgs "list"; + delete = pkgs: ageScript pkgs "delete"; + fixup = pkgs: ageScript pkgs "fixup"; + deploy.local = pkgs: ageScript pkgs "deploy-local"; + deploy.remote = lib.mkIf (cfg.ssh.target != null) ( + pkgs: + pkgs.writeScript "deploy-remote" '' + #!/bin/sh + set -euo pipefail + ${ageScript pkgs "deploy"} | ssh "${cfg.ssh.target}" -i "${cfg.ssh.identity}" ' + # set -euo pipefail # <- Can't do this; the shell might not be bash :( + mkdir -p "${cfg.targetDirectory}.tmp" + tar xf - -C "${cfg.targetDirectory}.tmp" + mv "${cfg.targetDirectory}.tmp" -T "${cfg.targetDirectory}" + ' + '' + ); + + fileModule = + { secret, name, ... }: + { + path = "${cfg.targetDirectory}/${secret.name}/${name}"; + }; + }; + + # TODO: write a service that decrypts the files at runtime! (or something...) +} diff --git a/nixos/modules/security/secrets/example/common/backend-age.py b/nixos/modules/security/secrets/example/common/backend-age.py new file mode 100644 index 0000000000000..b8d86be8a80da --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-age.py @@ -0,0 +1,152 @@ +import tempfile +import sys +import os +import subprocess +import argparse +import json +import tarfile +from pathlib import Path + +parser = argparse.ArgumentParser(description="secrets-age-backend") +parser.add_argument("config") + +subparsers = parser.add_subparsers(title="commands", dest="command", required=True) + +get_parser = subparsers.add_parser("get") +get_parser.add_argument("generator") +get_parser.add_argument("filename") + +set_parser = subparsers.add_parser("set") +set_parser.add_argument("generator") +set_parser.add_argument("filename") + +list_parser = subparsers.add_parser("list") + +delete_parser = subparsers.add_parser("delete") +delete_parser.add_argument("generator") +delete_parser.add_argument("filename") + +fixup_parser = subparsers.add_parser("fixup") +fixup_parser.add_argument("filelist") + +deploy_local_parser = subparsers.add_parser("deploy-local") +deploy_local_parser.add_argument("system_root", type=Path) + +deploy_parser = subparsers.add_parser("deploy") + +args = vars(parser.parse_args()) + +with open(args["config"]) as f: + config = json.loads(f.read()) + +host_directory = Path(config["hostDirectory"]) +target_directory = Path(config["targetDirectory"]) + + +def host_secret_path(generator, filename): + return host_directory / "generators" / generator / "files" / filename + + +def target_secret_path(generator, filename): + return target_directory / generator / filename + + +def list_host_secrets(): + out = [] + if host_directory.exists(): + for generator in (host_directory / "generators").iterdir(): + for file in (generator / "files").iterdir(): + out.append((generator.name, file.name)) + return out + + +def get_secret(generator, file, out_path): + if generator in config["generators"]: + identity = config["generators"][generator]["identity"]["host"] + else: + identity = config["identity"]["host"] + + command = [ + "age", + "--decrypt", + "--identity", + identity, + "--output", + out_path, + host_secret_path(generator, file), + ] + + subprocess.run(command, check=True) + + +def set_secret(generator, filename, in_path): + out_path = host_secret_path(generator, filename) + out_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + + if generator in config["generators"]: + pub_keys = config["generators"][generator]["publicKeys"] + else: + pub_keys = config["publicKeys"] + + command = ["age", "--encrypt", "--output", out_path] + for pub_key in pub_keys: + command += ["--recipient", pub_key] + command += [in_path] + + subprocess.run(command, check=True) + + +def parse_file_list(lines): + pairs = [] + for line in lines: + if not line: + continue + + parts = line.strip().split() + if len(parts) != 2: + raise Exception(f"Malformed file list line: {line}") + + [generator, filename] = parts + pairs.append((generator, filename)) + + return pairs + + +if args["command"] == "get": + generator = args["generator"] + get_secret(generator, args["filename"], os.environ["out"]) +elif args["command"] == "set": + set_secret(args["generator"], args["filename"], os.environ["in"]) +elif args["command"] == "list": + for generator, filename in list_host_secrets(): + print(f"{generator} {filename}") +elif args["command"] == "delete": + host_secret_path(args["generator"], args["filename"]).unlink(missing_ok=True) +elif args["command"] == "fixup": + with tempfile.NamedTemporaryFile() as fp: + fp = Path(fp.name) + for generator, filename in parse_file_list(args["filelist"].split("\n")): + get_secret(generator, filename, fp) + set_secret(generator, filename, fp) +elif args["command"] == "deploy-local": + sys_root = args["system_root"] + if not sys_root.exists(): + raise Exception(f"Directory '{sys_root}' does not exist") + for generator, filename in parse_file_list(sys.stdin): + in_path = host_secret_path(generator, filename) + if not in_path.exists(): + raise Exception(f"Missing secret file '{generator}/{filename}'") + + out_path = target_secret_path(generator, filename) + # TODO: is there a better way to do this? + out_path = Path(f"{sys_root}{out_path}") + out_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + in_path.copy(out_path) +elif args["command"] == "deploy": + with tarfile.open("sample.tar.gz", "w|", fileobj=sys.stdout.buffer) as tar: + for generator, filename in parse_file_list(sys.stdin): + in_path = host_secret_path(generator, filename) + if not in_path.exists(): + raise Exception(f"Missing secret file '{generator}/{filename}'") + + tar.add(in_path, arcname=f"{generator}/{filename}") diff --git a/nixos/modules/security/secrets/example/common/backend-plain.nix b/nixos/modules/security/secrets/example/common/backend-plain.nix new file mode 100644 index 0000000000000..51c83e7876c77 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-plain.nix @@ -0,0 +1,74 @@ +# An example plain-text secret backend written in Python. +{ config, lib, ... }: +let + cfg = config.secrets.settings.store.plain; + + # We bake the configuration and required command into the script that calls + # the CLI. I'm not sure if doing it this way is better than overriding the + # Python writer directly. I guess this method shared the original Python + # script derivation, but that might not be meaningful for such a small script. + backendScript = + pkgs: command: + let + backendJSONConfig = pkgs.writeText "plain.json" ( + builtins.toJSON { + inherit (cfg) hostDirectory targetDirectory; + } + ); + + scriptSource = builtins.readFile ./backend-plain.py; + + raw = pkgs.writers.writePython3Bin "secrets-plain-backend" { + flakeIgnore = [ + "W191" + "E501" + ]; + } scriptSource; + in + lib.getExe ( + pkgs.symlinkJoin { + name = "secrets-plain-backend"; + paths = [ raw ]; + buildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/secrets-plain-backend \ + --add-flags "${backendJSONConfig} ${command}" + ''; + } + ); +in +{ + options.secrets.settings.store.plain = { + hostDirectory = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets-ng-ng-plain/host/${config.networking.hostName}"; + description = '' + The directory where the plain backend will store the secrets on the + host machine. + ''; + }; + + targetDirectory = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets-ng-ng-plain/target"; + description = '' + The directory where the plain backend will store the secrets on the + target machine. + ''; + }; + }; + + config.secrets.backends.store.plain = { + get = pkgs: backendScript pkgs "get"; + set = pkgs: backendScript pkgs "set"; + list = pkgs: backendScript pkgs "list"; + delete = pkgs: backendScript pkgs "delete"; + deploy.local = pkgs: backendScript pkgs "deploy-local"; + + fileModule = + { secret, name, ... }: + { + path = "${cfg.targetDirectory}/${secret.name}/${name}"; + }; + }; +} diff --git a/nixos/modules/security/secrets/example/common/backend-plain.py b/nixos/modules/security/secrets/example/common/backend-plain.py new file mode 100644 index 0000000000000..ff7e3a0aab8ac --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-plain.py @@ -0,0 +1,87 @@ +import sys +import os +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser(description="secrets-plain-backend") +parser.add_argument("config") + +subparsers = parser.add_subparsers(title="commands", dest="command", required=True) + +get_parser = subparsers.add_parser("get") +get_parser.add_argument("generator") +get_parser.add_argument("filename") + +set_parser = subparsers.add_parser("set") +set_parser.add_argument("generator") +set_parser.add_argument("filename") + +list_parser = subparsers.add_parser("list") + +delete_parser = subparsers.add_parser("delete") +delete_parser.add_argument("generator") +delete_parser.add_argument("filename") + +deploy_local_parser = subparsers.add_parser("deploy-local") +deploy_local_parser.add_argument("system_root", type=Path) + +args = vars(parser.parse_args()) + +with open(args["config"]) as f: + config = json.loads(f.read()) + +host_directory = Path(config["hostDirectory"]) +target_directory = Path(config["targetDirectory"]) + + +def host_secret_path(generator, filename): + return host_directory / "generators" / generator / "files" / filename + + +def target_secret_path(generator, filename): + return target_directory / generator / filename + + +def parse_file_list(lines): + pairs = [] + for line in lines: + if not line: + continue + + parts = line.strip().split() + if len(parts) != 2: + raise Exception(f"Malformed file list line: {line}") + + [generator, filename] = parts + pairs.append((generator, filename)) + return pairs + + +if args["command"] == "get": + host_secret_path(args["generator"], args["filename"]).copy(os.environ["out"]) +elif args["command"] == "set": + out_path = host_secret_path(args["generator"], args["filename"]) + out_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + Path(os.environ["in"]).copy(out_path) +elif args["command"] == "list": + if host_directory.exists(): + for generator in (host_directory / "generators").iterdir(): + for file in (generator / "files").iterdir(): + print(f"{generator.name} {file.name}") +elif args["command"] == "delete": + host_secret_path(args["generator"], args["filename"]).unlink(missing_ok=True) +elif args["command"] == "deploy-local": + sys_root = args["system_root"] + if not sys_root.exists(): + raise Exception(f"Directory '{sys_root}' does not exist") + for generator, filename in parse_file_list(sys.stdin): + in_path = host_secret_path(generator, filename) + if not in_path.exists(): + raise Exception(f"Missing secret file '{generator}/{filename}'") + + out_path = target_secret_path(generator, filename) + # TODO: is there a better way to do this? + out_path = Path(f"{sys_root}{out_path}") + out_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + in_path.copy(out_path) diff --git a/nixos/modules/security/secrets/example/common/backend-prompt-simple.nix b/nixos/modules/security/secrets/example/common/backend-prompt-simple.nix new file mode 100644 index 0000000000000..3d70905857329 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-prompt-simple.nix @@ -0,0 +1,36 @@ +# An example interactive prompt backend written in Bash. +let + mkScript = + name: text: pkgs: + pkgs.lib.getExe ( + pkgs.writeShellApplication { + inherit name text; + runtimeInputs = [ pkgs.coreutils ]; + checkPhase = ""; + } + ); +in +{ + secrets.backends.prompt.simple.ask = mkScript "prompt" '' + out=''${out:?} # Make shellcheck happy + + prompt="$4" + if [[ ! -z "$5" ]]; then + prompt="$prompt ($5)" + fi + + if [[ "$3" == "line" ]]; then + read -rp "$prompt: " text + echo -n "$text" > "$out" + elif [[ "$3" == "hidden" ]]; then + read -srp "$prompt: " text + echo "" + echo -n "$text" > "$out" + elif [[ "$3" == "multiline" ]]; then + echo "<$prompt>" > "$out" + $EDITOR "$out" + else + exit 1 + fi + ''; +} diff --git a/nixos/modules/security/secrets/example/common/backend-prompt-test.nix b/nixos/modules/security/secrets/example/common/backend-prompt-test.nix new file mode 100644 index 0000000000000..eed346675baa5 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/backend-prompt-test.nix @@ -0,0 +1,31 @@ +# An example non-interactive prompt backend which merely reads the files from a +# static directory. +{ config, lib, ... }: +let + cfg = config.secrets.settings.prompt.test; + + mkScript = + name: text: pkgs: + pkgs.lib.getExe ( + pkgs.writeShellApplication { + inherit name text; + runtimeInputs = [ pkgs.coreutils ]; + checkPhase = ""; + } + ); +in +{ + options.secrets.settings.prompt.test.inputDirectory = lib.mkOption { + type = lib.types.oneOf [ + lib.types.str + lib.types.path + ]; + description = '' + The directory where the plain-text prompt inputs should be read from. + ''; + }; + + config.secrets.backends.prompt.test.ask = mkScript "prompt" '' + cp ${cfg.inputDirectory}/"$1"/"$2" "$out" + ''; +} diff --git a/nixos/modules/security/secrets/example/common/key-host.txt b/nixos/modules/security/secrets/example/common/key-host.txt new file mode 100644 index 0000000000000..01152af848dc8 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/key-host.txt @@ -0,0 +1,3 @@ +# created: 2026-07-14T16:25:42+02:00 +# public key: age13ar5t7vvsssmckjhjtngy3p5y0v4k896ecjrxveql9ysu8gxhe9sdar3k3 +AGE-SECRET-KEY-1CFFCTZ6AAG82DKYU0SNKDKHQ9FCYJM6CJNTNMT5AE78255JHH2CS33SSVC diff --git a/nixos/modules/security/secrets/example/common/key-target.txt b/nixos/modules/security/secrets/example/common/key-target.txt new file mode 100644 index 0000000000000..2e3cebb371c59 --- /dev/null +++ b/nixos/modules/security/secrets/example/common/key-target.txt @@ -0,0 +1,3 @@ +# created: 2026-07-15T15:00:22+02:00 +# public key: age195x33zrqzppjfnj2rjjlq3z8s64r5zlwe6rcywm9zu6agf449pmqdslyat +AGE-SECRET-KEY-1F32QQWYAPQFF76RQ4Q8ZLLCSYCZKEED89A8A880YT4LDJZZ2SGAQQHM3G9 diff --git a/nixos/modules/security/secrets/example/flake/flake.lock b/nixos/modules/security/secrets/example/flake/flake.lock new file mode 100644 index 0000000000000..bd5197e9fbbce --- /dev/null +++ b/nixos/modules/security/secrets/example/flake/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1786393488, + "narHash": "sha256-WFP4dchjwBn9Gf806sxrVA1OWsbGqpP/uwUHYPvX+aw=", + "owner": "starlitcanopy", + "repo": "nixpkgs", + "rev": "fa1db85d35c981df697a23dc27d226b63b63a75e", + "type": "github" + }, + "original": { + "owner": "starlitcanopy", + "ref": "master", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nixos/modules/security/secrets/example/flake/flake.nix b/nixos/modules/security/secrets/example/flake/flake.nix new file mode 100644 index 0000000000000..48defc481f312 --- /dev/null +++ b/nixos/modules/security/secrets/example/flake/flake.nix @@ -0,0 +1,21 @@ +{ + inputs.nixpkgs.url = "github:starlitcanopy/nixpkgs?ref=master"; + + outputs = inputs: { + nixosConfigurations.example = inputs.nixpkgs.lib.nixosSystem { + system = "aarch64-linux"; + + modules = [ ../basic-config ]; + }; + + # Example of how one can override the host package set! + secretsConfigurations.differentArch = + let + pkgsHost = inputs.nixpkgs.legacyPackages.x86_64-linux; + in + pkgsHost.nixos-secrets.jsonify { + inherit pkgsHost; + configuration = inputs.self.nixosConfigurations.example; + }; + }; +} diff --git a/nixos/modules/security/secrets/example/structured-ids/README.md b/nixos/modules/security/secrets/example/structured-ids/README.md new file mode 100644 index 0000000000000..0e80203b9264a --- /dev/null +++ b/nixos/modules/security/secrets/example/structured-ids/README.md @@ -0,0 +1 @@ +This directory shows a proof of concept of one way to tack structured IDs on top of the existing NixOS module. Do note that a more advanced solution would be writing a completely new Nix(OS) module together with its own `jsonify`-esque function that handles the ID generation. diff --git a/nixos/modules/security/secrets/example/structured-ids/default.nix b/nixos/modules/security/secrets/example/structured-ids/default.nix new file mode 100644 index 0000000000000..4c3238707f4fc --- /dev/null +++ b/nixos/modules/security/secrets/example/structured-ids/default.nix @@ -0,0 +1,116 @@ +{ + config, + lib, + modulesPath, + ... +}: +let + hashId = config.secrets.settings.id.hasher; +in +{ + imports = [ + "${modulesPath}/security/secrets" + ../common/backend-plain.nix + ../common/backend-age.nix + ../common/backend-prompt-simple.nix + ./module.nix + ]; + + secrets = { + backends.defaults.store = "plain"; + backends.defaults.prompt = "simple"; + + settings.store.age.publicKeys = [ + "age13ar5t7vvsssmckjhjtngy3p5y0v4k896ecjrxveql9ysu8gxhe9sdar3k3" # Host + "age195x33zrqzppjfnj2rjjlq3z8s64r5zlwe6rcywm9zu6agf449pmqdslyat" # Target + ]; + + # NOTE: do *not* do this with real keys!!! This will copy the keys to the + # world-readable Nix store, which is most probably not what you want! + settings.store.age.identity.host = toString ../common/key-host.txt; + settings.store.age.identity.target = toString ../common/key-target.txt; + + settings.store.age.ssh.target = "root@lapetus.overlay.moonythm.dev"; + settings.store.age.ssh.identity = "/home/moon/.ssh/id_ed25519"; + + store.example = { + id = { + group = "first"; + name = "example"; + }; + + prompts.example = { + label = "Your name"; + description = "the person to address the greeting to"; + type = "multiline"; + }; + + files.example.deploy = false; + + generate = + pkgs: + pkgs.writeScript "gen-example" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + echo "Hewwo $(cat "$prompts/example")!" > "$out/example" + ''; + }; + + store.derived = + let + dep = hashId { + group = "first"; + name = "example"; + }; + in + { + backend = "age"; + + id = { + group = "first"; + name = "derived"; + }; + + dependencies = [ dep ]; + files.derived = { }; + + generate = + pkgs: + pkgs.writeScript "gen-derived" '' + #!/bin/sh + export PATH="${ + lib.makeBinPath [ + pkgs.coreutils + pkgs.cowsay + ] + }" + cat $in/${dep}/example | cowsay > $out/derived + ''; + }; + + store.derivedPlain = + let + dep = hashId { + group = "first"; + name = "derived"; + }; + in + { + id = { + group = "second"; + name = "derived"; + }; + + dependencies = [ dep ]; + files.derived = { }; + + generate = + pkgs: + pkgs.writeScript "gen-derived-plain" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + cat $in/${dep}/derived > $out/derived + ''; + }; + }; +} diff --git a/nixos/modules/security/secrets/example/structured-ids/module.nix b/nixos/modules/security/secrets/example/structured-ids/module.nix new file mode 100644 index 0000000000000..c0338d3c3f6b6 --- /dev/null +++ b/nixos/modules/security/secrets/example/structured-ids/module.nix @@ -0,0 +1,31 @@ +{ lib, ... }: +let + hashId = id: toString (builtins.hashString "md5" (builtins.toJSON id)); +in +{ + options.secrets.settings.id.hasher = lib.mkOption { + type = lib.types.functionTo lib.types.str; + default = hashId; + readOnly = true; + }; + + options.secrets.store = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { config, ... }: + { + options.id = lib.mkOption { + type = lib.types.json; + description = "A freeform ID that can be used to identify the generator"; + example = { + foo = 1; + goo = 2; + }; + }; + + config.name = hashId config.id; + } + ) + ); + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 029a2543ec935..61fcd30c2fdb3 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1190,6 +1190,7 @@ in nixos-rebuild-target-host = runTest { imports = [ ./nixos-rebuild-target-host.nix ]; }; + nixos-secrets-basic-generators = runTest ./nixos-secrets/basic-generators; nixpkgs = pkgs.callPackage ../modules/misc/nixpkgs/test.nix { inherit evalMinimalConfig; }; nixpkgs-config-allow-unfree = pkgs.callPackage ../modules/misc/nixpkgs/test-nixpkgs-config-allow-unfree.nix diff --git a/nixos/tests/nixos-secrets/basic-generators/config/common.nix b/nixos/tests/nixos-secrets/basic-generators/config/common.nix new file mode 100644 index 0000000000000..3cdac59a01ca0 --- /dev/null +++ b/nixos/tests/nixos-secrets/basic-generators/config/common.nix @@ -0,0 +1,17 @@ +{ modulesPath, ... }: +{ + imports = [ + "${modulesPath}/security/secrets/example/common/backend-prompt-test.nix" + "${modulesPath}/security/secrets/example/common//backend-plain.nix" + "${modulesPath}/security/secrets" + ]; + + secrets = { + backends.defaults.prompt = "test"; + settings.prompt.test.inputDirectory = ./prompt-inputs; + + backends.defaults.store = "plain"; + settings.store.plain.hostDirectory = "/tmp/secrets-demo"; + settings.store.plain.targetDirectory = "/tmp/secrets-demo"; + }; +} diff --git a/nixos/tests/nixos-secrets/basic-generators/config/config1.nix b/nixos/tests/nixos-secrets/basic-generators/config/config1.nix new file mode 100644 index 0000000000000..807e4ce3b1286 --- /dev/null +++ b/nixos/tests/nixos-secrets/basic-generators/config/config1.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +{ + imports = [ ./common.nix ]; + + secrets = { + store.greeting = { + prompts.name.description = "Your name"; + + files.greeting = { }; + generate = + pkgs: + pkgs.writeScript "gen-greeting" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + echo "Hewwo $(cat "$prompts/name")!" > $out/greeting + ''; + }; + + store.derived = { + dependencies = [ "greeting" ]; + files.derived = { }; + generate = + pkgs: + pkgs.writeScript "gen-derived" '' + #!/bin/sh + export PATH="${ + lib.makeBinPath [ + pkgs.coreutils + pkgs.cowsay + ] + }" + cat $in/greeting/greeting | cowsay > $out/derived + ''; + }; + }; +} diff --git a/nixos/tests/nixos-secrets/basic-generators/config/config2.nix b/nixos/tests/nixos-secrets/basic-generators/config/config2.nix new file mode 100644 index 0000000000000..8ccd1a897847e --- /dev/null +++ b/nixos/tests/nixos-secrets/basic-generators/config/config2.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +{ + imports = [ ./common.nix ]; + + secrets = { + store.greeting = { + prompts.name.description = "Your name"; + files.greeting = { }; + generate = + pkgs: + pkgs.writeScript "gen-example" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + echo "Hewwo $(cat "$prompts/name")!!" > $out/greeting + ''; + }; + + store.derived = { + dependencies = [ "greeting" ]; + files.derived2 = { }; + generate = + pkgs: + pkgs.writeScript "gen-derived" '' + #!/bin/sh + export PATH="${ + lib.makeBinPath [ + pkgs.coreutils + pkgs.cowsay + ] + }" + + cat $in/greeting/greeting | cowsay > $out/derived2 + ''; + }; + }; +} diff --git a/nixos/tests/nixos-secrets/basic-generators/config/prompt-inputs/greeting/name b/nixos/tests/nixos-secrets/basic-generators/config/prompt-inputs/greeting/name new file mode 100644 index 0000000000000..48cdce8528724 --- /dev/null +++ b/nixos/tests/nixos-secrets/basic-generators/config/prompt-inputs/greeting/name @@ -0,0 +1 @@ +placeholder diff --git a/nixos/tests/nixos-secrets/basic-generators/default.nix b/nixos/tests/nixos-secrets/basic-generators/default.nix new file mode 100644 index 0000000000000..cc48d430be191 --- /dev/null +++ b/nixos/tests/nixos-secrets/basic-generators/default.nix @@ -0,0 +1,68 @@ +{ + name = "nixos-secrets-basic-generators"; + + nodes.machine = + { pkgs, ... }: + { + nix.nixPath = [ "nixpkgs=${pkgs.path}" ]; + environment.systemPackages = [ pkgs.nixos-secrets ]; + environment.etc."nixos".source = ./config; + + system.extraDependencies = [ + (import ../collect-secrets-scripts.nix { + inherit pkgs; + configuration = ./config/config1.nix; + }) + (import ../collect-secrets-scripts.nix { + inherit pkgs; + configuration = ./config/config2.nix; + }) + ]; + }; + + testScript = '' + machine.wait_for_unit("default.target") + + # We ensure this works even if there's nothing there to garbage collect + machine.succeed("nixos-secrets collect-garbage -f /etc/nixos/config1.nix") + + # Generating the first config + machine.succeed("nixos-secrets generate -f /etc/nixos/config1.nix") + t.assertEqual("Hewwo placeholder!", machine.succeed("cat /tmp/secrets-demo/generators/greeting/files/greeting").strip()) + t.assertIn("< Hewwo placeholder! >", machine.succeed("cat /tmp/secrets-demo/generators/derived/files/derived")) + + # Switching to the second config + machine.succeed("nixos-secrets collect-garbage -f /etc/nixos/config2.nix") + machine.succeed("test ! -f /tmp/secrets-demo/generators/derived/files/derived") + machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix") + t.assertIn("< Hewwo placeholder! >", machine.succeed("cat /tmp/secrets-demo/generators/derived/files/derived2")) + + # Should work without a sandbox + machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix") + t.assertIn("< Hewwo placeholder! >", machine.succeed("cat /tmp/secrets-demo/generators/derived/files/derived2")) + + # Should be a no-op (there's no garbage to collect) + machine.succeed("nixos-secrets collect-garbage -f /etc/nixos/config2.nix ") + + # "derived" depends on "greeting" + t.assertIn("Successfully updated 1 secret(s)", machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix -g derived")) + t.assertIn("Successfully updated 2 secret(s)", machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix -g greeting")) + t.assertIn("Successfully updated 2 secret(s)", machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix -g greeting -g derived")) + + # Local deployments + machine.succeed("mkdir /tmp/system") + machine.succeed("nixos-secrets deploy -l /tmp/system -f /etc/nixos/config2.nix") + t.assertIn("Hewwo placeholder!!", machine.succeed("cat /tmp/system/tmp/secrets-demo/greeting/greeting")) + t.assertIn("< Hewwo placeholder!! >", machine.succeed("cat /tmp/system/tmp/secrets-demo/derived/derived2")) + + # --set + machine.succeed("mkdir /tmp/greeting-files") + machine.succeed("echo 'green orange' > /tmp/greeting-files/greeting") + machine.succeed("nixos-secrets generate -f /etc/nixos/config2.nix --set greeting=/tmp/greeting-files") + t.assertIn("green orange", machine.succeed("cat /tmp/secrets-demo/generators/greeting/files/greeting")) + t.assertIn("< green orange >", machine.succeed("cat /tmp/secrets-demo/generators/derived/files/derived2")) + + # This script is written to always fail! + machine.fail("nixos-secrets deploy -f /etc/nixos/config2.nix") + ''; +} diff --git a/nixos/tests/nixos-secrets/collect-secrets-scripts.nix b/nixos/tests/nixos-secrets/collect-secrets-scripts.nix new file mode 100644 index 0000000000000..2efacd78d4adf --- /dev/null +++ b/nixos/tests/nixos-secrets/collect-secrets-scripts.nix @@ -0,0 +1,31 @@ +# The secrets CLI needs to call Nix at runtime. This would usually fail inside a +# NixOS test, as the VM has no network access. +# +# This module takes a NixOS configuration, collects every secrets-related +# derivation, and returns their combined closures. Said closure can later be +# added to `system.extraDependencies`. +{ pkgs, configuration }: +let + inherit (pkgs) lib; + evaluated = pkgs.nixos-secrets.jsonify { + inherit configuration; + pkgsHost = pkgs; + pkgsTarget = pkgs; + }; + derivations = [ + (lib.mapAttrsToList (_: x: [ + x.delete + x.deploy.local + x.deploy.remote + x.fixup + x.get + x.list + x.set + ]) evaluated.backends.store) + (lib.mapAttrsToList (_: x: x.ask) evaluated.backends.prompt) + (lib.mapAttrsToList (_: x: x.generate) evaluated.store) + ]; +in +pkgs.closureInfo { + rootPaths = lib.lists.filter (x: x != null) (lib.lists.flatten derivations); +} diff --git a/pkgs/by-name/ni/nixos-secrets/.gitignore b/pkgs/by-name/ni/nixos-secrets/.gitignore new file mode 100644 index 0000000000000..bee8a64b79a99 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/pkgs/by-name/ni/nixos-secrets/docs/01-secrets.md b/pkgs/by-name/ni/nixos-secrets/docs/01-secrets.md new file mode 100644 index 0000000000000..799721861837b --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/docs/01-secrets.md @@ -0,0 +1,189 @@ +# NixOS-secrets + +NixOS-secrets is meant to provide a unified interface secret handling tools similar to [agenix](https://github.com/ryantm/agenix) and [sops-nix](https://github.com/mic92/sops-nix) can be built around. + +## Secrets and files + +A _secret_ represents a collection of files, usually (but not necessarily) generated by a script. In case the generation is not meant to be done programmatically, the files can also be imported manually (more on this later). A secret's generation script can depend on both other secrets and on prompts that ask the user for input. + +The files associated with each secret must be handled by a so-called _store backend_. A single `nixos-secrets` configuration can contain multiple store backends. For now, think of backends as little programs that handle the storage and retrieval (and possibly also deployment!) of the secrets one puts in. + +There are many ways to provide a nixos-secrets configuration to the CLI, but for now we'll start with the most basic of them all. That is, by using the provided [NixOS module](../../../../nixos/modules/security/secrets/default.nix). The module must be imported manually: + +```nix +{ modulesPath, ... }: +{ + imports = [ "${modulesPath}/security/secrets" ]; +} +``` + +Here's how a basic generator might look like: + +```nix +{ + secrets.store.user = { + backend = "plain"; + files.greeting = { }; + generate = + pkgs: + pkgs.writeScript "gen-user" '' + #!/bin/sh + echo "Hewwo world!" > "$out/greeting" + ''; + }; +} +``` + +For now, think of the backend field as a black box. We'll go over how backends work (and how you can implement your own!) a bit later. + +The `files.greeting = { }` tells the CLI that the `user` secret contains a file called `greeting`. We set it to an empty object, as there's no additional configuration we want to provide. + +One important thing to note is that we never directly set `generate` to the path of a script. Indeed, we instead created a function from a package set to the given script. This might seem a bit odd at first — don't NixOS modules already have access to `pkgs` via the module system? They do! Still, the generator scripts might need to run on a different architecture than the one the NixOS system is about to be deployed to. As a result, scripts must use the package set they are given as an argument. + +Scripts (be it generator or backend scripts) can be written in any language. Throughout this document we will be mostly using Bash, although that's not a hard requirement for using `nixos-secrets` (indeed, the [example backends](../../../../nixos/modules/security/secrets/example/common) are written in Python, for example!). + +### Generating secrets using the CLI + +The CLI accepts a few different kinds of inputs. The two most important ones are `--file` and `--flake`. We'll be using the former in this document, although the latter works very similarly. To generate the secret described above, one can use the `generate` command: + +``` +$ nixos-secrets generate --file /path/to/config.nix +Updating 'user' (missing metadata) +Generating 'user' +Successfully updated 1 secret(s) +Running fixup scripts: +- Skipping 'plain' (no fixup script) +``` + +Note that this will do nothing on further re-runs (the secrets already exist, after all!). To force a generator to be re-run, one can use the `--generate` flag: + +``` +$ nixos-secrets generate --file /path/to/config --generate user +Updating 'user' (forced) +Generating 'user' +Successfully updated 1 secret(s) +Running fixup scripts: +- Skipping 'plain' (no fixup script) +``` + +### Secret dependencies + +Generators can depend on other secrets. The inputs for a given generator will be available in the `$in` directory for the respective script. For example: + +```nix +{ + secrets.store.derived = { + dependencies = [ "user" ]; + files.cow-greeting = { }; + script = + pkgs: + pkgs.writeScript "gen-derived" '' + #!/bin/sh + export PATH="${ + lib.makeBinPath [ + pkgs.coreutils + pkgs.cowsay + ] + }" + + cat $in/user/greeting | cowsay > $out/cow-greeting + ''; + }; +} +``` + +Note that we did not manually specify a backend this time! In such scenarios, the module will automatically use the `secrets.backends.defaults.store` backend. Running the CLI again will generate the new secret: + +``` +$ nixos-secrets generate --file /path/to/config +Skipping 'user' +Updating 'derived' (missing metadata) +Generating 'derived' +Successfully updated 1 secret(s) +Running fixup scripts: +- Skipping 'plain' (no fixup script) +``` + +### Importing secrets from files + +Secrets can also be imported from files on disk. The files must be arranged in a directory that matches the structure of the files declared in the user's config: + +``` +$ mkdir manual +$ echo "Per aspera ad astra" > manual/greeting +``` + +We can now use the `--set` flag to import the directory we've just created: + +``` +$ nixos-secrets generate --file /path/to/config --set user=manual +Updating 'user' (forced) +Importing 'user' from disk +Updating 'derived' (dependencies changed: user) +Generating 'derived' +Successfully updated 2 secret(s) +Running fixup scripts: +- Skipping 'plain' (no fixup script) +``` + +Note that the CLI has automatically detected the need for `derived` to also be regenerated! + +## Prompts + +It is often useful for generators to allow human (or otherwise external) input. This can be achieved using _prompts_. As for generators, prompts are also provided by a set of customizable backends: + +```nix +{ + secrets.store.user = { + prompts.name = { + backend = "simple"; + label = "Your name"; + description = "The person to address the greeting to"; + type = "line"; + }; + + files.greeting = { }; + + generate = + pkgs: + pkgs.writeScript "gen-user" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + echo "Hi! My name is $(cat "$prompts/name")." > "$out/greeting" + ''; + }; +} +``` + +The `label` and `description` fields offer the user information about what prompt they are currently filling in. The `type` field tells the backend what kind of input they should ask the user for. The supported types are currently `line`, `hidden`, and `multiline`. + +The `backend` field works similarly to the `backend` field used by secrets (although prompt and store backends are distinct concepts!). If absent, the module will default to the `secrets.backends.defaults.prompt` option. + +Running the above through the CLI (with the `--generate` flag!) will ask the user to type in their name before generating the rest of the secrets: + +``` +$ nixos-secrets generate --file /path/to/config --generate user +Updating 'user' (forced) +Evaluating prompts for 'user': +- 'name' +Generating 'user' +Updating 'derived' (dependencies changed: user) +Generating 'derived' +Successfully updated 2 secret(s) +Running fixup scripts: +- Skipping 'plain' (no fixup script) +``` + +Note that a prompt backend could choose to return data from any external source, be it a hard-coded string (for testing) or some kind of GUI. + +### Prompt storage + +Prompts are ethereal, and do not get stored on disk. If prompt storage is desired, then one can wrap the prompt in a "dummy" secret that simply copies its value into a file. Other secrets can then depend on said "dummy" secret. + +The aforementioned technique can also be used for sharing a prompt across multiple secrets. + +Alternatively, one could write a prompt backend that caches the given values for a certain period of time. The possibilities are endless! + +## Sandboxing + +Generator scripts are sandboxed by default (one can disable this by passing `--no-sandbox`), although one should not rely on this instead of checking the scripts themselves beforehand. A malicious script could, for example, generate an intentionally "weak" secret without ever leaving the confines of the sandbox! diff --git a/pkgs/by-name/ni/nixos-secrets/docs/02-prompt-backends.md b/pkgs/by-name/ni/nixos-secrets/docs/02-prompt-backends.md new file mode 100644 index 0000000000000..39d1fe1303a3d --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/docs/02-prompt-backends.md @@ -0,0 +1,47 @@ +## Prompt backends + +Before jumping in to the more complex idea of store backends, we'll go over how one can specify prompt backends. Prompt backends require a single `ask` script, which is responsible for asking the user for information. The script is given four arguments: + +1. The secret name (`user` in the previous example) +2. The prompt name (`name` in the previous example) +3. The prompt type +4. The prompt label +5. The prompt description + +The first two arguments are not necessarily meant to be displayed to the user. Instead, they're provided for use in scenarios like testing, such that the backend can identify precisely which prompt it is currently answering to (since labels & descriptions need not be unique!). + +A very simple prompt backend would look something like this: + +```nix +{ + secrets.backends.prompt.simple.ask = + pkgs: + pkgs.writeScript "simple-prompt" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + + # We do not care about $1 (the secret name) nor $2 (the prompt name). + + prompt="$4" + if [[ ! -z "$5" ]]; then + prompt="$prompt ($5)" + fi + + if [[ "$3" == "line" ]]; then + read -rp "$prompt: " text + echo -n "$text" > "$out" + elif [[ "$3" == "hidden" ]]; then + read -srp "$prompt: " text + echo "" + echo -n "$text" > "$out" + elif [[ "$3" == "multiline" ]]; then + echo "<$prompt>" > "$out" + $EDITOR "$out" + else + exit 1 + fi + ''; +} +``` + +One thing to note is that prompt (and by extension, store) backends will not be sandboxed (unlike generator scripts). diff --git a/pkgs/by-name/ni/nixos-secrets/docs/03-store-backends.md b/pkgs/by-name/ni/nixos-secrets/docs/03-store-backends.md new file mode 100644 index 0000000000000..e3619ffe16810 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/docs/03-store-backends.md @@ -0,0 +1,136 @@ +## Store/generator backends + +Store backends are specified in a similar manner to prompt backends: + +```nix +{ + # For a full example, see the provided examples + secrets.backends.store.plain = { }; +} +``` + +### Get + +Unlike prompt backends, store backends must provide a number of different scripts. The most basic of said scripts are `get` and `set`. The former is given the secret name and the file name as an argument, and must return the content of said backend to `$out`. The CLI needs to be able to access any of the secrets at runtime in order for secret dependencies to work out. The `get` script can be omitted as long as the backend in question is never used as a dependency for another generator. + +```nix +{ + secrets.backends.store.plain.get = + pkgs: + pkgs.writeScript "secrets-plain-get" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + cat /var/lib/nixos-secrets-plain/generators/"$1"/files/"$2" > "$out" + ''; +} +``` + +### Set + +The `set` script, on the other hand, is mandatory. As with the `get` script, the `set` script is given a secret name and a file name as an argument, together with a secret at `$in`, and is responsible for saving said secret for later. + +```nix +{ + secrets.backends.store.plain.set = + pkgs: + pkgs.writeScript "secrets-plain-set" '' + #!/bin/sh + export PATH="${lib.makeBinPath [ pkgs.coreutils ]}" + cat "$in" > /var/lib/nixos-secrets-plain/generators/"$1"/files/"$2" + ''; +} +``` + +### Garbage collection + +The user might remove secrets from their configuration, yet the respective secrets will still exist on disk. The CLI offers the `collect-garbage` command for handling this exact scenario. In order to support garbage collection, a backend must provide the `list` and `delete` scripts. + +The `delete` script is given a secret and a file name as an argument, and must delete the given file (you might notice a theme here). + +The `list` argument must print (to standard out) a list of every file it currently finds on disk (or in the cloud, or wherever the secrets are stored). The format of the output goes as follows: every line contains a space separated list where the first element is the secret name, and the second element is the file name (_not_ the full file path!). For example, the output of `list` could look like this: + +```txt +gen-foo file-1 +gen-foo file-2 +gen-bar file-3 +gen-goo file-4 +``` + +These operations are not difficult to implement in the easy case of managing a single machine. One thing to note is that backends might want to share secrets across multiple machines. The aforementioned `list` command must then only return secrets related to the given machine/configuration! (otherwise the CLI will ask the backend to delete every one of the files it finds in the list but not in the given machine/configuration). + +### Performing automatic updates + +Backends might need to perform maintenance work on the secret files on disk. Think re-keying when using `age` keys and a new recipient is added, or perhaps rotating API keys when storing the secrets remotely. A backend can provide a `fixup` script, which will be run after each `generate` command. This script will be run regardless of whether any of the files involved got updated/regenerated. + +Said script can perform side effects, yet must remain idempotent. The script is expected to perform the necessary updates to every file within a single invocation, and is given a list of files to act on as argument, in the same format as the output of the `list` script (although the actual content might of course be different). + +### Deployment + +Backends can also provide `deploy.local` and `deploy.remote` scripts. Note that each secret file has a `deploy` flag that is on by default. Users can choose to disable deployment for any of their secrets by setting said flag to `false`. This might be useful (for example) when handling secrets that are only meant to be used as inputs to other generators, but must not have their outputs deployed right away. A backend can choose to not provide either of the scripts above. + +The names of the two scripts might give away their intended purpose. The former is meant for deploying the secrets to a system that has its system root mounted to the current machine's filesystem, and will therefore receive a path to the system root as its first argument. The latter is meant for deploying to fully remote systems. + +As an example, the former will usually receive `/` as its first argument (when deploying to the current machine), although this is not always the case. For example, consider a live CD where `nixos-install` has finished running, yet the newly constructed machine is not currently running either (and thus cannot be accessible over SSH or whatnot). + +Both scripts receive a list of secrets to deploy, via standard input. The list is given in the same format used for the output of the `list` script (although the actual content might of course be different). Deployment scripts do not follow the pattern of taking in a secret and a file name as arguments. This is because batched operations are preferred in scenarios like that of a person using passphrase-protected SSH keys or touch-protected hardware keys. + +### Output paths + +Last but not least, backends can set an optional `fileModule` that will be imported by each file corresponding to the given backend. This module is usually responsible for setting the `path` attribute, representing the location the file will be deployed to on the target machine (this can, for example, be referenced from other parts of the given NixOS config). + +For example, the plain backend might work as follows: + +```nix +{ + secrets.backends.store.plain.fileModule = + { secret, name, ... }: + { + path = "${config.secrets.settings.store.plain.targetDirectory}/${secret.name}/${name}"; + }; +} +``` + +### Per-backend options + +Backends might require additional configuration (e.g. where should the files go on the host machine? What keys should they be encrypted with? Etc). While a backend is free to put those options anywhere (backends are full-blown NixOS modules, after all!), the convention is to put them under `secrets.settings.store.*` and `secrets.settings.prompt.*` respectively. + +### Per-secret or per-file backend options + +Backends will commonly need to define custom per-generator or per-file options. While the latter can already be achieved with the aforementioned `fileModule`, the former needs to currently be done by hand. Since the NixOS module system merges submodules defined in the same location, one can achieve the above as follows: + +```nix +{ + # Extracted from the age backend + options.secrets.store = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options.age.identity.host = lib.mkOption { + default = config.secrets.age.identity.host; + type = lib.types.str; + description = '' + Path to the age private key file for decryption on the host machine + ''; + }; + } + ); + }; +} +``` + +### Environment variables + +Backends receive additional environment variables one can read when things like Git-root detection are required. In particular, `NIXOS_SECRETS_FLAKE` will contain the value passed to `--flake` and `NIXOS_SECRETS_CONFIG` will contain the path given to `--json` or `--file` respectively. + +### Available backends + +We are not currently planning to ship a production-ready backend alongside the CLI (although this might change once the CLI stabilizes). The goal of the interface is to be lean enough such that anyone can write a simple backend meeting their needs in their language of choice. Still, four (currently somewhat scuffed) example backends can be found in [`example/common`](../../../../nixos/modules/security/secrets/example/common). + +### Metadata + +The `nixos-secrets` will attach additional metadata to each secret. The metadata is there in order to detect dependency changes, recover from crashes mid-generation, and so on. The metadata is stored in a file named `.nixos-secrets-metadata`. Store backends do not require special logic/scripts for handling metadata files. Indeed, to a backend, the metadata is merely another file associated with the given secret (although one the user hasn't manually declared). + +### Failure modes + +The aforementioned metadata system should protect one's secrets from most crashes. Still, this system is not perfect. In particular, spooky things might happen if multiple instances of the CLI are invoked simultaneously (we should perhaps consider some sort of locking mechanism in the future, although that would complicate things a lot, especially when the CLI's instances are run from separate machines). + +More importantly, a backend's `set` script should perform the update in an atomic matter, when possible. The metadata only being partially written could cause issues for future runs of the program (although it will most likely cause the given secret to be regenerated). diff --git a/pkgs/by-name/ni/nixos-secrets/docs/04-schema.md b/pkgs/by-name/ni/nixos-secrets/docs/04-schema.md new file mode 100644 index 0000000000000..40c5c75f15b86 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/docs/04-schema.md @@ -0,0 +1,56 @@ +## The secrets schema + +Throughout this section, we'll use "unevaluated NixOS configuration" to refer to configuration files containing NixOS modules (think `/etc/nixos/configuration.nix`). On the other hand, a "(pre-)evaluated NixOS configuration" is one that has already been passed to `` (or `nixpkgs.lib.nixosSystem` when using flakes). Moreover, the "target" system is the one the secrets are meant to be deployed to, and the "host" system is the one where the CLI is being run. + +Earlier on we observed that the secrets CLI can take in NixOS configurations as argument. Of course, this by itself can be read in multiple ways. For example — do the configurations in question need to be evaluated already? If not, where is Nixpkgs taken from? + +The CLI does accept pre-evaluated configurations (like one would, for example, expect when using flakes). When given a non-evaluated configuration, the CLI will evaluate it using the Nixpkgs available in the `NIX_PATH`. One should pre-evaluate their configuration if pinning of the Nixpkgs version is desired. + +Once the configuration is evaluated, the CLI will extract the data it needs into a Nix attrset that can be directly serialized as JSON (this implies the package set-reliant script functions are evaluated with the host package set as an argument, for example) before being taken in by the Python code. + +This is done by calling to the so-called [`jsonify.nix`](./nixos_secrets/nix/jsonify.nix) function. If we were to assign a type signature to the aforementioned function, it would (barring some internal arguments) look something like this: + +``` +jsonify : { config, pkgsHost ? null, pkgsTarget ? null } -> SecretsConfiguration +``` + +We've already seen what happens if `config` is given as a (possibly evaluated) NixOS configuration. If this function receives a `SecretsConfiguration` as the config argument, then the function will simply return the configuration it is given. This means one can side-step the `jsonify.nix` logic entirely and produce a Nix attrset containing the needed data by whichever means they desire (for example, as part of a tool that's totally disconnected from the NixOS module system). + +The `SecretsConfiguration` type is documented as a JSON schema in [`secrets-config.schema.json`](./src/nixos_secrets/secrets-config.schema.json). + +One can also sidestep going through Nix-lang altogether by using the `--json` flag to pass a JSON string satisfying the aforementioned schema + +### Using separate host and target NixOS instances + +One can pre-evaluate their NixOS configuration in order to pin the target's Nixpkgs instance. This will implicitly also set the host's Nixpkgs instance to the same value. If this is not desired (for example, when the target and host architectures differ), then one must invoke `jsonify` themselves, passing its output to the CLI (one could also side-step `jsonify` entirely, as explained above; a person doing that is assumed to already know what they're doing though!). + +One can achieve the above as follows (do note that flakes are not necessary for this! I chose to provide a flakes-based example since none of the examples above used them): + +```nix +{ + inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + + outputs = inputs: { + nixosConfigurations.example = inputs.nixpkgs.lib.nixosSystem { + system = "aarch64-linux"; + modules = [ + # The config goes here + ]; + }; + + secretsConfigurations.differentArch = + let + pkgsHost = inputs.nixpkgs.legacyPackages.x86_64-linux; + in + pkgsHost.nixos-secrets.jsonify { + inherit pkgsHost; + configuration = inputs.self.nixosConfigurations.example; + + # Observe that pkgsTarget is not needed, as the given configuration has + # been # pre-evaluated! + }; + }; +} +``` + +One would then pass the configuration to the CLI like usual (in this case, via `nixos-secrets --flake path/to/flake#secretsConfiguration.differentArch`). diff --git a/pkgs/by-name/ni/nixos-secrets/package.nix b/pkgs/by-name/ni/nixos-secrets/package.nix new file mode 100644 index 0000000000000..016eb77b34d68 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/package.nix @@ -0,0 +1,62 @@ +{ + lib, + mkShell, + nixosTests, + python3, + python3Packages, + ruff, + makeWrapper, + bubblewrap, + jsonschema, +}: + +python3Packages.buildPythonApplication { + __structuredAttrs = true; + + name = "nixos-secrets"; + format = "pyproject"; + nativeBuildInputs = [ + python3Packages.setuptools + makeWrapper + ]; + src = ./src; + + postFixup = '' + wrapProgram $out/bin/nixos-secrets \ + --prefix PATH : ${ + lib.makeBinPath [ + bubblewrap + jsonschema + ] + } + ''; + + passthru.devShell = mkShell { + packages = [ + python3 + bubblewrap + ruff + jsonschema + ]; + }; + + passthru.tests = { + inherit (nixosTests) + nixos-secrets-basic-generators + ; + }; + + # This is perhaps not a good idea? + passthru.jsonify = import ./src/nixos_secrets/nix/jsonify.nix; + + meta = { + description = "NixOS secret management abstraction"; + homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/by-name/ni/nixos-secrets"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ + lassulus + prescientmoon + ]; + mainProgram = "nixos-secrets"; + }; +} diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/__init__.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/__main__.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/__main__.py new file mode 100644 index 0000000000000..9ae637f13cd52 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + main() diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/args.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/args.py new file mode 100644 index 0000000000000..b7b11482450cf --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/args.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Any, Self, Mapping, List +from .error import SecretsError + + +@dataclass(frozen=True) +class SecretsArgs: + file: Optional[Path] + flake: Optional[str] + json: Optional[str] + attr: Optional[str] + disable_sandbox: bool + dry_run: bool + local: Optional[str] # "deploy" only + generators: List[str] # "generate" only + set: Mapping[str, Path] # "generate" only + timeout: Optional[float] # "generate" only + command: str # gotta figure out how to type this properly + verbose: str + + def from_dict(d: Mapping[str, Any]) -> Self: + # This one is only in the dict when the "generate" command is used. + # I wish we had proper sum types... + if "generators" not in d: + d["generators"] = [] + if "local" not in d: + d["local"] = None + if "timeout" not in d: + d["timeout"] = None + + setArgDict = dict() + if "set" in d: + for arg in d["set"]: + try: + key, value = arg.split("=") + if key in setArgDict: + raise SecretsError( + f"Multiple --set arguments received for generator '{key}'" + ) + setArgDict[key] = Path(value) + except ValueError: + raise SecretsError(f"--set expects key=value pairs: '{arg}' given") + d["set"] = setArgDict + + args = SecretsArgs(**d) + + configSources = [] + + if args.file is not None: + configSources.append(args.file) + if args.flake is not None: + configSources.append(args.flake) + if args.json is not None: + configSources.append(args.json) + + if len(configSources) != 1: + raise SecretsError( + "Precisely one of the --file, --flake, or --json flags must be provided" + ) + + if args.attr and not args.file: + raise SecretsError("--attr is only supported for --file") + + setGenerators = set(args.set.keys()) + forcedGenerators = set(args.generators) + + if overlappingGenerators := setGenerators.intersection(forcedGenerators): + overlappingList = ", ".join(sorted(overlappingGenerators)) + raise SecretsError( + f"A generator cannot be passed to both --generate and --set, yet the following have been: {overlappingList}" + ) + + return args diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/cli.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/cli.py new file mode 100644 index 0000000000000..b9efc1bb33412 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/cli.py @@ -0,0 +1,139 @@ +import argparse +import sys +from pathlib import Path +from .error import SecretsError +from .eval import evaluate_config +from .generate import generate_secrets +from .gc import collect_garbage +from .args import SecretsArgs +from .deploy import deploy + + +def common_args(parser: argparse.ArgumentParser): + parser.add_argument( + "-f", + "--file", + type=Path, + default=None, + metavar="", + help="Path to config.nix", + ) + + parser.add_argument( + "-F", + "--flake", + type=str, + default=None, + metavar="", + help="Flake-based alternative to --file", + ) + + parser.add_argument( + "-j", + "--json", + type=str, + default=None, + metavar="", + help="Parse the secrets configuration from the given JSON file", + ) + + parser.add_argument( + "-A", + "--attr", + type=str, + metavar="", + help="Gets the attribute at the given path", + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would be done without executing", + ) + + parser.add_argument( + "--no-sandbox", + action="store_true", + dest="disable_sandbox", + help="Do not run the generator scripts inside bubblewrap", + ) + + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Print all the logs generated by the various scripts", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="nixos-secrets CLI") + + # Global arguments + subparsers = parser.add_subparsers(title="commands", dest="command", required=True) + + eval_parser = subparsers.add_parser("evaluate") + + common_args(eval_parser) + gen_parser = subparsers.add_parser("generate", help="(Re)generate secrets") + common_args(gen_parser) + gen_parser.add_argument( + "-g", + "--generate", + dest="generators", + metavar="", + type=str, + default=[], + action="append", + help="Generator(s) to force the regeneration of.", + ) + gen_parser.add_argument( + "-s", + "--set", + dest="set", + metavar="", + type=str, + default=[], + action="append", + help="Generator(s) to import from external files.", + ) + gen_parser.add_argument( + "-t", + "--timeout", + metavar="", + type=float, + default=10, + help="The amount of seconds to wait for individual generator scripts to run.", + ) + + gc_parser = subparsers.add_parser("collect-garbage", help="Delete stale secrets") + + common_args(gc_parser) + deploy_parser = subparsers.add_parser("deploy", help="Deploy secrets") + common_args(deploy_parser) + deploy_parser.add_argument( + "-l", + "--local", + type=str, + metavar="", + help="Deploy to a locally attached filesystem", + ) + + args = parser.parse_args() + args = SecretsArgs.from_dict(vars(args)) + + try: + config = evaluate_config(args) + if args.command == "evaluate": + print(config) + elif args.command == "generate": + generate_secrets(args, config) + elif args.command == "collect-garbage": + collect_garbage(args, config) + elif args.command == "deploy": + deploy(args, config) + else: + raise SecretsError(f"Command '{args.command}' is not implemented :(") + except SecretsError as e: + print(str(e), file=sys.stderr) + exit(1) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/config.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/config.py new file mode 100644 index 0000000000000..3ef6c9845392e --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/config.py @@ -0,0 +1,195 @@ +from dataclasses import dataclass + +from typing import Mapping, List, Any, Set, Self, Optional +from .error import SecretsError +import re + + +safe_name_regex = re.compile("^[a-zA-Z0-9:_\\.-]+$") +meta_file_name = ".nixos-secrets-metadata" # In this file to prevent cyclic imports + + +@dataclass(frozen=True) +class SecretsPromptBackend: + name: str + ask: str + + def from_json(name: str, json: Any) -> Self: + return SecretsPromptBackend(name=name, ask=json["ask"]) + + +@dataclass(frozen=True) +class SecretsPrompt: + name: str + label: str + description: Optional[str] + backend: str + type: str # There's probably a way to type this properly.. + + def from_json(name: str, json: Any) -> Self: + return SecretsPrompt( + name=name, + label=json["label"], + description=json["description"], + backend=json["backend"], + type=json["type"], + ) + + +@dataclass(frozen=True) +class SecretsStoreBackend: + name: str + get: Optional[str] + set: str + delete: Optional[str] + list: Optional[str] + fixup: Optional[str] + deployRemote: Optional[str] + deployLocal: Optional[str] + + def from_json(name: str, json: Any) -> Self: + return SecretsStoreBackend( + name=name, + get=json["get"], + set=json["set"], + delete=json.get("delete"), + list=json.get("list"), + fixup=json.get("fixup"), + deployRemote=json["deploy"].get("remote"), + deployLocal=json["deploy"].get("local"), + ) + + +@dataclass(frozen=True) +class SecretsFile: + name: str + deploy: bool = False + + def from_json(name: str, json: Any) -> Self: + if safe_name_regex.search(name) is None: + raise SecretsError( + f"File '{name}' does not have a valid name. Currently, only alphanumeric characters, dashes, underscores, and dots are allowed." + ) + + if name == meta_file_name: + raise SecretsError(f"Files cannot use the reserved name '{meta_file_name}'") + + return SecretsFile(name=name, deploy=json["deploy"]) + + +@dataclass(frozen=True) +class SecretsSecret: + name: str + backend: str + generate: Optional[str] + dependencies: List[str] + prompts: Mapping[str, SecretsPrompt] + files: Mapping[str, SecretsFile] + + def from_json(name: str, json: Any) -> Self: + if safe_name_regex.search(name) is None: + raise SecretsError( + f"Secret '{name}' does not have a valid name. Currently, only alphanumeric characters, dashes, underscores, and dots are allowed." + ) + + result = SecretsSecret( + name=name, + backend=json["backend"], + generate=json["generate"], + dependencies=json["dependencies"], + prompts={}, + files={}, + ) + + for k, v in json["prompts"].items(): + result.prompts[k] = SecretsPrompt.from_json(k, v) + + for k, v in json["files"].items(): + result.files[k] = SecretsFile.from_json(k, v) + + if not result.files: + raise SecretsError(f"Secret '{name}' has no associated files") + + if result.generate is None and result.dependencies: + raise SecretsError( + f"Secret '{name}' has associated dependencies without a corresponding generator script" + ) + + if result.generate is None and result.prompts: + raise SecretsError( + f"Secret '{name}' has associated prompts without a corresponding generator script" + ) + + return result + + +@dataclass(frozen=True) +class SecretsConfig: + generators: Mapping[str, SecretsSecret] + storeBackends: Mapping[str, SecretsStoreBackend] + promptBackends: Mapping[str, SecretsPromptBackend] + + def from_json(json: Any) -> Self: + result = SecretsConfig(generators={}, storeBackends={}, promptBackends={}) + + for k, v in json["backends"]["prompt"].items(): + result.promptBackends[k] = SecretsPromptBackend.from_json(k, v) + + for k, v in json["backends"]["store"].items(): + result.storeBackends[k] = SecretsStoreBackend.from_json(k, v) + + for k, v in json["store"].items(): + result.generators[k] = SecretsSecret.from_json(k, v) + + referencedGenerators: Set[str] = set() + referencedStoreBackends: Set[str] = set() + referencedPromptBackends: Set[str] = set() + + for name, gen in result.generators.items(): + referencedGenerators.update(gen.dependencies) + referencedStoreBackends.add(gen.backend) + + for secret in result.generators.values(): + for name, prompt in secret.prompts.items(): + referencedPromptBackends.add(prompt.backend) + + if missingPromptBackends := referencedPromptBackends - set( + result.promptBackends.keys() + ): + missingList = ", ".join(sorted(missingPromptBackends)) + raise SecretsError( + f"The following prompt backends are referenced but not defined: {missingList}" + ) + + if missingStoreBackends := referencedStoreBackends - set( + result.storeBackends.keys() + ): + missingList = ", ".join(sorted(missingStoreBackends)) + raise SecretsError( + f"The following generator backends are referenced but not defined: {missingList}" + ) + + if missingGenerators := referencedGenerators - set(result.generators.keys()): + missingList = ", ".join(sorted(missingGenerators)) + raise SecretsError( + f"The following generators are referenced but not defined: {missingList}" + ) + + return result + + def files_for_backend( + self: Self, + backend: SecretsStoreBackend, + deployed_only: bool = False, + ) -> List[tuple[str, str]]: + files = [] + for generator in self.generators.values(): + if generator.backend != backend.name: + continue + + for file in generator.files.values(): + # Checks that deployed_only implies file.deploy + if not deployed_only or deployed_only and file.deploy: + files.append((generator.name, file.name)) + + return files diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/deploy.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/deploy.py new file mode 100644 index 0000000000000..fa40db3c6e8b2 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/deploy.py @@ -0,0 +1,21 @@ +from .args import SecretsArgs +from .config import SecretsConfig +from .generate import generate_secrets +from .exec import deploy_secrets + + +def deploy(args: SecretsArgs, config: SecretsConfig): + generate_secrets(args, config) + + print(f"Running deploy scripts for {len(config.storeBackends)} backends:") + + for backend in config.storeBackends.values(): + files = config.files_for_backend(backend, deployed_only=True) + + if not files: + print(f"- Skipping '{backend.name}' (no files to deploy)") + continue + + print(f"- '{backend.name}' ({len(files)} file(s))") + if not args.dry_run: + deploy_secrets(args, config, backend, files) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/error.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/error.py new file mode 100644 index 0000000000000..70c1d0a39cced --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/error.py @@ -0,0 +1,4 @@ +class SecretsError(Exception): + """Base exception for nixos-secrets errors.""" + + pass diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/eval.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/eval.py new file mode 100644 index 0000000000000..1a022efb6d18b --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/eval.py @@ -0,0 +1,96 @@ +import json +import subprocess +import functools +from pathlib import Path +from typing import Any, Optional +from .error import SecretsError +from .args import SecretsArgs +from .config import SecretsConfig + +jsonify_path = Path(__file__).parent / "nix" / "jsonify.nix" +with open(jsonify_path) as f: + jsonify_source = f.read() + +schema_path = Path(__file__).parent / "secrets-config.schema.json" + + +def evaluate_config(args: SecretsArgs) -> SecretsConfig: + json_str = evaluate_config_raw(args) + + try: + subprocess.run( + ["jv", schema_path, "-"], + input=json_str, + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise SecretsError(f"Config validation error:\n{e.stdout}") + + return SecretsConfig.from_json(json.loads(json_str)) + + +def evaluate_config_raw(args: SecretsArgs) -> Any: + if args.json is not None: + try: + with open(args.json) as f: + return f.read() + except json.decoder.JSONDecodeError as e: + raise SecretsError(f"Error parsing JSON: {e}") + elif args.flake is not None: + expr = f"configuration: ({jsonify_source}) {{ inherit configuration; }}" + evalCommand = [ + "nix", + "eval", + "--json", + args.flake, + "--apply", + expr, + ] + + if args.verbose: + evalCommand.append("--show-trace") + elif args.file is not None: + expr = f""" +({jsonify_source}) {{ + configuration = (import {args.file.resolve()}){"" if args.attr is None else f".{args.attr}"}; + pkgsDefault = import {nixpkgs_path()} {{}}; +}} +""" + evalCommand = [ + "nix-instantiate", + "--eval", + "--json", + "--strict", + "--expr", + "--read-write-mode", + expr, + ] + + try: + result = subprocess.run( + evalCommand, + capture_output=True, + text=True, + check=True, + ) + + return result.stdout + except subprocess.CalledProcessError as e: + raise SecretsError(f"Error evaluating nix expression:\n{e.stderr}") + + +@functools.cache +def nixpkgs_path() -> Optional[str]: + try: + result = subprocess.run( + ["nix-instantiate", "--find-file", "nixpkgs"], + capture_output=True, + text=True, + check=True, + ) + + return result.stdout.strip() + except subprocess.CalledProcessError: + return None diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/exec.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/exec.py new file mode 100644 index 0000000000000..15c20f1348db5 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/exec.py @@ -0,0 +1,296 @@ +import os +import subprocess +import graphlib +import functools +from typing import List, Set, Mapping +from pathlib import Path +from .config import ( + SecretsConfig, + SecretsStoreBackend, + SecretsSecret, + SecretsFile, + SecretsPrompt, +) +from .error import SecretsError +from .args import SecretsArgs + + +# Prevents the terminal from getting stuck in an invalid state (i.e. no echo) after, for example, a generator script has finished running. +def reset_terminal_state(): + # https://stackoverflow.com/questions/7938402/terminal-in-broken-state-invisible-text-no-echo-after-exit-during-input + # os.system("stty sane") + # The above breaks the nixos-test runner, so I disabled it for now + pass + + +@functools.cache +def build_binary(path: Path) -> Path: + try: + result = subprocess.run( + ["nix-store", "--realise", path], + capture_output=True, + text=True, + check=True, + ) + + return result.stdout.strip() + except subprocess.CalledProcessError as e: + raise SecretsError(f"Error building '{path}':\n{e.stderr}") + + +def backend_env_vars(args: SecretsArgs) -> Mapping[str, str]: + out = dict() + if args.json: + out["NIXOS_SECRETS_CONFIG"] = args.json + elif args.file: + out["NIXOS_SECRETS_CONFIG"] = args.file + elif args.flake: + out["NIXOS_SECRETS_FLAKE"] = args.flake + return out + + +def get_secret( + args: SecretsArgs, + config: SecretsConfig, + generator: SecretsSecret, + file: SecretsFile, + out: Path, +): + backend = config.storeBackends[generator.backend] + if backend.get is None: + raise SecretsError( + f"Backend '{backend.name}' has no 'get' script, yet the generator ''{generator.name}' requires one" + ) + + binary = build_binary(backend.get) + try: + env = os.environ.copy() + env["out"] = out + env.update(backend_env_vars(args)) + subprocess.run( + [binary, generator.name, file.name], + capture_output=not args.verbose, + env=env, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error getting secret '{generator.name}/{file.name}' via the '{backend.name}' backend:\n{e.stderr}" + ) + finally: + reset_terminal_state() + + +def set_secret( + args: SecretsArgs, + config: SecretsConfig, + generator: SecretsSecret, + file: SecretsFile, + at: Path, +): + backend = config.storeBackends[generator.backend] + binary = build_binary(backend.set) + try: + env = os.environ.copy() + env["in"] = at + env.update(backend_env_vars(args)) + subprocess.run( + [binary, generator.name, file.name], + capture_output=not args.verbose, + env=env, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error setting secret '{generator.name}/{file.name}' via the '{backend.name}' backend:\n{e.stderr}" + ) + finally: + reset_terminal_state() + + +# NOTE: we take strings here instead of proper SecretsBackend/SecretsFile object since +# the secrets to be deleted might no longer exist in the configuration (e.g. +# while garbage collecting). +def delete_secret( + args: SecretsArgs, + config: SecretsConfig, + backend: SecretsStoreBackend, + gen_name: str, + file_name: str, +): + binary = build_binary(backend.delete) + try: + env = os.environ.copy() + env.update(backend_env_vars(args)) + subprocess.run( + [binary, gen_name, file_name], + capture_output=not args.verbose, + env=env, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error deleting secret '{gen_name}/{file_name}' via the '{backend.name}' backend:\n{e.stderr}" + ) + finally: + reset_terminal_state() + + +def list_secrets( + args: SecretsArgs, config: SecretsConfig, backend: SecretsStoreBackend +) -> Set[tuple[str, str]]: + binary = build_binary(backend.list) + try: + pairs = set() + env = os.environ.copy() + env.update(backend_env_vars(args)) + result = subprocess.run( + [binary], + capture_output=True, + text=True, + check=True, + env=env, + ) + + for line in result.stdout.strip().split("\n"): + if not line: + continue + + parts = line.strip().split() + if len(parts) == 2: + pairs.add((parts[0], parts[1])) + else: + raise SecretsError( + f"Malformed output for list script in backend '{backend.name}': {line}" + ) + + return pairs + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error listing secrets for the '{backend.name}' backend:\n{e.stderr}" + ) + + +def deploy_secrets( + args: SecretsArgs, + config: SecretsConfig, + backend: SecretsStoreBackend, + files: List[tuple[str, str]], +): + inputLines = [] + for generator, filename in files: + inputLines.append(f"{generator} {filename}") + + local = args.local is not None + script = backend.deployLocal if local else backend.deployRemote + if script is None: + scriptName = "deploy.local" if local else "deploy.remote" + raise SecretsError(f"Backend '{backend.name}' has no '{scriptName}' script") + + binary = build_binary(script) + try: + env = os.environ.copy() + env.update(backend_env_vars(args)) + subprocess.run( + [binary, args.local] if local else [binary], + input="\n".join(inputLines), + capture_output=not args.verbose, + env=env, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error running deploy via the '{backend.name}' backend:\n{e.stderr}" + ) + finally: + reset_terminal_state() + + +def run_prompt( + args: SecretsArgs, + config: SecretsConfig, + secret: SecretsSecret, + prompt: SecretsPrompt, + out: Path, +): + backend = config.promptBackends[prompt.backend] + binary = build_binary(backend.ask) + try: + env = os.environ.copy() + env["out"] = out + env.update(backend_env_vars(args)) + command = [binary, secret.name, prompt.name, prompt.type, prompt.label] + if prompt.description: + command.append(prompt.description) + subprocess.run( + command, + env=env, + check=True, + ) + except subprocess.CalledProcessError: + raise SecretsError( + f"Error running prompt '{prompt.name}' via the '{backend.name}' backend" + ) + finally: + reset_terminal_state() + + +def execution_order(config: SecretsConfig) -> List[str]: + ts = graphlib.TopologicalSorter() + + for name, gen in config.generators.items(): + ts.add(name, *gen.dependencies) + + try: + return list(ts.static_order()) + except Exception as e: + raise SecretsError(f"Dependency cycle detected in configuration:\n{e}") + + +def fixup_all(args: SecretsArgs, config: SecretsConfig): + print("Running fixup scripts:") + + errors = [] + for backend in config.storeBackends.values(): + files = config.files_for_backend(backend) + + if not backend.fixup: + print(f"- Skipping '{backend.name}' (no fixup script)") + continue + elif not files: + print(f"- Skipping '{backend.name}' (no files)") + continue + else: + print(f"- '{backend.name}' ({len(files)} file(s))") + + if args.dry_run: + continue + + inputLines = [] + for generator, filename in files: + inputLines.append(f"{generator} {filename}") + + binary = build_binary(backend.fixup) + try: + env = os.environ.copy() + env.update(backend_env_vars(args)) + subprocess.run( + [binary, "\n".join(inputLines)], + capture_output=not args.verbose, + env=env, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + errors.append( + f"Error running fixup script for backend '{backend.name}':\n{e.stderr}" + ) + finally: + reset_terminal_state() + + if errors: + raise SecretsError("\n".join(errors)) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/gc.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/gc.py new file mode 100644 index 0000000000000..0419e85054541 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/gc.py @@ -0,0 +1,35 @@ +from .args import SecretsArgs +from .config import SecretsConfig, meta_file_name +from .exec import list_secrets, delete_secret, fixup_all + + +def collect_garbage(args: SecretsArgs, config: SecretsConfig): + for backend in config.storeBackends.values(): + if not backend.list or not backend.delete: + print(f"Skipping '{backend.name}': missing 'list' or 'delete' script") + + continue + + secrets = list_secrets(args, config, backend) + specified = set() + for generator in config.generators.values(): + for file in generator.files.values(): + specified.add((generator.name, file.name)) + + unspecified = secrets - specified + if not unspecified: + print(f"Skipping '{backend.name}': nothing to collect") + continue + + print(f"Backend '{backend.name}':") + for gen_name, file_name in sorted(unspecified): + if file_name == meta_file_name: + continue + + if args.dry_run: + print(f"- Would delete '{gen_name}/{file_name}'") + else: + print(f"- Deleting '{gen_name}/{file_name}'") + delete_secret(args, file, backend, gen_name, file_name) + + fixup_all(args, config) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/generate.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/generate.py new file mode 100644 index 0000000000000..27291f096da3f --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/generate.py @@ -0,0 +1,273 @@ +import os +import tempfile +import subprocess +from uuid import uuid4 +from pathlib import Path +from typing import Mapping + +from .args import SecretsArgs +from .config import SecretsConfig, SecretsSecret +from .meta import SecretsMetadata, VersionID, get_meta, set_meta +from .exec import ( + execution_order, + build_binary, + get_secret, + set_secret, + fixup_all, + run_prompt, + reset_terminal_state, +) +from .error import SecretsError +from .list import build_file_list + + +def generate_secrets(args: SecretsArgs, config: SecretsConfig): + forced_regens = set(args.generators + list(args.set.keys())) + for gen_name in forced_regens: + if gen_name not in config.generators: + raise SecretsError(f"Invalid secret name '{gen_name}'") + + order = execution_order(config) + files = build_file_list(args, config) + + updated = 0 + + # Bubblewrap requires usernamespaces to be enabled, so it won't work (by + # default) in places like Ubuntu. At @Qubasa's suggestion, I have thus made it + # so a very simple bwrap invocation is used to "test the waters" before running + # the actual generator scripts inside a sandbox. + if not args.disable_sandbox: + try: + subprocess.run( + [ + "bwrap", + "--ro-bind", + "/", + "/", + "echo", + ], + capture_output=True, + check=True, + text=True, + timeout=1, + ) + except subprocess.CalledProcessError, subprocess.TimeoutExpired: + raise SecretsError( + "Bubblewrap is not available. Either get bubblewrap working or retry with --no-sandbox." + ) + + # NOTE: we are going to run the scripts inside bubblewrap, thus sharing a + # single temporary directory is not a concern. + with tempfile.TemporaryDirectory() as temp: + temp = Path(temp) + + up_to_date_meta = {} + for entry in order: + generator = config.generators[entry] + + # The regeneration logic goes as follows: + # - we regenerate if the user tells us to (via --set or --generate)... + # - ...or if any dependencies were added/removed... + # - ...or if any of the dependencies have themselves changed... + # - ...or if any of the files are missing + + regen = False + meta = get_meta(args, config, files, generator) + if entry in forced_regens: + print(f"Updating '{entry}' (forced)") + regen = True + elif meta: + meta_deps = set(meta.dependencies.keys()) + config_deps = set(generator.dependencies) + removed_deps = meta_deps - config_deps + added_deps = config_deps - meta_deps + if removed_deps: + dep_str = ", ".join(sorted(removed_deps)) + print(f"Updating '{entry}' (removed dependencies: {dep_str})") + regen = True + elif added_deps: + dep_str = ", ".join(sorted(added_deps)) + print(f"Updating '{entry}' (added dependencies: {dep_str})") + regen = True + else: + changed_deps = set() + for dep, id in meta.dependencies.items(): + if id != up_to_date_meta[dep].id: + changed_deps.add(dep) + if changed_deps: + dep_str = ", ".join(sorted(changed_deps)) + print(f"Updating '{entry}' (dependencies changed: {dep_str})") + regen = True + else: + print(f"Updating '{entry}' (missing metadata)") + regen = True + + if not regen: + for file in generator.files.values(): + if not files.has(generator.backend, generator.name, file.name): + print(f"Updating '{entry}' (file '{file.name}' is missing)") + regen = True + break + + if not regen: + print(f"Skipping '{entry}'") + up_to_date_meta[entry] = meta + continue + + in_dir = temp / "generators" / entry / "in" + os.makedirs(in_dir) + + out_dir = temp / "generators" / entry / "out" + os.makedirs(out_dir) + + prompt_in_dir = temp / "generators" / entry / "prompts" + os.makedirs(prompt_in_dir) + + if generator.prompts and entry not in args.set: + print(f"Evaluating prompts for '{entry}':") + for prompt in generator.prompts.values(): + print(f"- '{prompt.name}'") + if not args.dry_run: + run_prompt( + args, + config, + generator, + prompt, + prompt_in_dir / prompt.name, + ) + + if entry in args.set: + print(f"Importing '{entry}' from disk") + if args.dry_run: + continue + + set_files_from_dir(args, config, generator, args.set[entry]) + elif generator.generate is not None: + binary = build_binary(generator.generate) + + print(f"Generating '{entry}'") + if args.dry_run: + continue + + for dep_name in generator.dependencies: + dep = config.generators[dep_name] + os.makedirs(in_dir / dep_name) + for file in dep.files.values(): + try: + get_secret( + args, + config, + dep, + file, + in_dir / dep_name / file.name, + ) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error getting '{dep_name}/{file.name}': {e.stderr}" + ) + + try: + env = os.environ.copy() + env["in"] = in_dir + env["out"] = out_dir + env["prompts"] = prompt_in_dir + + if args.disable_sandbox: + subprocess.run( + [binary], + env=env, + capture_output=not args.verbose, + check=True, + text=True, + timeout=args.timeout, + input="", + ) + else: + subprocess.run( + [ + "bwrap", + "--unshare-all", + "--die-with-parent", + "--ro-bind", + "/nix/store", + "/nix/store", + "--ro-bind", + "/bin", + "/bin", + "--ro-bind", + "/usr/bin", + "/usr/bin", + "--ro-bind", + in_dir, + in_dir, + "--ro-bind", + prompt_in_dir, + prompt_in_dir, + "--bind", + out_dir, + out_dir, + "--clearenv", + "--setenv", + "in", + in_dir, + "--setenv", + "out", + out_dir, + "--setenv", + "prompts", + prompt_in_dir, + binary, + ], + capture_output=not args.verbose, + check=True, + text=True, + input="", + timeout=args.timeout, + ) + except subprocess.CalledProcessError as e: + raise SecretsError(f"Error generating '{entry}': {e.stderr}") + except subprocess.TimeoutExpired: + raise SecretsError(f"Generator '{entry}' timed out") + finally: + reset_terminal_state() + + set_files_from_dir(args, config, generator, out_dir) + else: + raise SecretsError( + f"Secret '{entry}' has no generator script, nor a corresponding --set argument, and hence can not be updated." + ) + + new_id = str(uuid4()) + dep_ids: Mapping[str, VersionID] = {} + for dep_name in generator.dependencies: + dep_ids[dep_name] = up_to_date_meta[dep_name].id + meta = SecretsMetadata(new_id, dep_ids) + + set_meta(args, config, generator, meta) + up_to_date_meta[entry] = meta + updated += 1 + + print(f"Successfully updated {updated} secret(s)") + + fixup_all(args, config) + + +def set_files_from_dir( + args: SecretsArgs, + config: SecretsConfig, + generator: SecretsSecret, + from_dir: Path, +): + for file in generator.files.values(): + if not (from_dir / file.name).exists(): + raise SecretsError( + f"Cannot update files for '{generator.name}': missing file '{file.name}'" + ) + + for file in generator.files.values(): + try: + set_secret(args, config, generator, file, from_dir / file.name) + except subprocess.CalledProcessError as e: + raise SecretsError( + f"Error setting '{generator.name}/{file.name}': {e.stderr}" + ) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/list.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/list.py new file mode 100644 index 0000000000000..35908190abf1d --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/list.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass +from typing import Set, Self +from .exec import list_secrets +from .config import SecretsConfig +from .args import SecretsArgs + + +# This is a workaround around the fact that tuples are not hashable +@dataclass(frozen=True) +class SecretsFileListEntry: + backend: str + secret: str + file: str + + +@dataclass(frozen=True) +class SecretsFileList: + entries: Set[SecretsFileListEntry] + + def has( + self: Self, + backend: str, + secret: str, + file: str, + ) -> bool: + return ( + SecretsFileListEntry(backend=backend, secret=secret, file=file) + in self.entries + ) + + +def build_file_list(args: SecretsArgs, config: SecretsConfig) -> SecretsFileList: + entries: Set[SecretsFileListEntry] = set() + for backend in config.storeBackends.values(): + for secret, file in list_secrets(args, config, backend): + entries.add(SecretsFileListEntry(backend.name, secret, file)) + return SecretsFileList(entries) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/meta.py b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/meta.py new file mode 100644 index 0000000000000..cae09a0e6977b --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/meta.py @@ -0,0 +1,73 @@ +import tempfile +import json +from dataclasses import dataclass +from typing import Any, Mapping, Self, Optional +from pathlib import Path + +from .error import SecretsError +from .args import SecretsArgs +from .exec import get_secret, set_secret +from .list import SecretsFileList +from .config import ( + SecretsConfig, + SecretsSecret, + SecretsFile, + meta_file_name, +) + +VersionID = str + + +@dataclass(frozen=True) +class SecretsMetadata: + id: VersionID + dependencies: Mapping[str, VersionID] + + def from_json(json: Any) -> Self: + return SecretsMetadata(id=json["id"], dependencies=json["dependencies"]) + + def to_json(self: Self): + return { + "version": 1, # Future proofing + "id": self.id, + "dependencies": self.dependencies, + } + + +def get_meta( + args: SecretsArgs, + config: SecretsConfig, + files: SecretsFileList, + secret: SecretsSecret, +) -> Optional[SecretsMetadata]: + if not files.has(secret.backend, secret.name, meta_file_name): + return None + + meta_file = SecretsFile(name=meta_file_name) + with tempfile.NamedTemporaryFile(mode="r") as file: + out_path = Path(file.name) + + try: + get_secret(args, config, secret, meta_file, out_path) + except SecretsError: + return None + + try: + raw = json.loads(file.read()) + return SecretsMetadata.from_json(raw) + except json.decoder.JSONDecodeError as e: + raise SecretsError(f"Error parsing metadata: {e}") + + +def set_meta( + args: SecretsArgs, + config: SecretsConfig, + secret: SecretsSecret, + meta: SecretsMetadata, +): + with tempfile.NamedTemporaryFile(mode="w") as file: + in_path = Path(file.name) + raw = json.dumps(meta.to_json()) + file.write(raw) + file.flush() + set_secret(args, config, secret, SecretsFile(name=meta_file_name), in_path) diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/nix/jsonify.nix b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/nix/jsonify.nix new file mode 100644 index 0000000000000..a5b5dd0a29d9b --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/nix/jsonify.nix @@ -0,0 +1,153 @@ +# Takes a configuration and prepares it to be consumed by the Python CLI. +# +# There's three kinds of supported configurations: +# - unevaluated NixOS configurations (will be evaluated with the target Nixpkgs +# instance) +# - evaluated NixOS configurations (the options will be extracted from the +# respective NixOS module) +# - evaluated secrets configurations (the script will then be a NO-OP), i.e. +# objects already matching the secrets schema (TODO: document this) +# +# Note that this file is *not* a Nix(OS) module! This will automatically be +# evaluated by the secrets CLI. +# +# Parameters: +# - `configuration`: The thing to actually turn into a secrets configuration. +# - `pkgsTarget`: the package set to use for the target system. Will fallback +# to the package set the configuration was evaluated with (for NixOS +# configurations) or to `pkgsDefault` otherwise. +# - `pkgsDefault`: a default for when the `pkgsTarget` is not otherwise +# specified. Passed from the CLI. +# - `pkgsHost`: the package set to use for the host system. Will fall back to +# the resolved package set for the target machine. +# +# People who want to write custom Nix(OS)-modules for use with the secrets CLI +# should write an accompanying `jsonify`-esque function, and pass its output to +# the CLI. As long as the output matches the secrets configuration schema (TODO: +# document that), this function will leave said output untouched and let the +# CLI do its thing. +{ + configuration, + pkgsTarget ? null, + pkgsHost ? null, + pkgsDefault ? null, +}: +let + pkgsTarget' = + if pkgsTarget != null then + pkgsTarget + else if configuration._type or null == "configuration" then + configuration.pkgs + else if pkgsDefault != null then + pkgsDefault + else + throw "Cannot infer the package set for the target machine."; + + pkgsHost' = if pkgsHost != null then pkgsHost else pkgsTarget'; + + inherit (pkgsHost') lib; + + # If the configuration has been evaluated already, simply keep it that way. + # Otherwise, evaluate it. + cfg = + if configuration._type or null == "configuration" then + configuration.config.secrets + else + (import (pkgsTarget'.path + "/nixos/lib/eval-config.nix") { + modules = [ configuration ]; + }).config.secrets; + + # Generator scripts might have to run on a different system from the target + # machine's. As a result, they are not merely packages, but functions from + # package sets to packages. + # + # Note that, when possible, we want to be "lazy" about building said + # derivations. In particular, we want Nix to realise the derivation files on + # disk, but not build them. This works pretty nicely out of the box when using + # the .drvPath attribute of a derivation. + # + # Still, some people might not pass a derivation here, but instead a + # non-top-level store-path (for example, "${pkgs.foo}/bin/goo"). We want to + # support this as well, so we detect such paths and wrap them in a tiny proxy + # script. This feels a bit hacky, but we couldn't find a better way to not + # throw away the string's context when converting it to JSON (for consumption + # from the Python side). + evalDeferredPackage = + pkg: + if pkg == null then + null + else + let + forced = pkg pkgsHost'; + drv = + if forced.type or null == "derivation" then + forced + else + pkgsHost'.writeScript "secrets-wrapper-script" '' + #!/bin/sh + exec ${forced} "$@" + ''; + in + drv.drvPath; +in +# We want this function to be idempotent. That is, we want running it repeatedly +# to produce the same result as only running it once. This is useful since +# advanced users might prefer to manually call this in order to override the +# various package sets involved. The Python CLI has no way of knowing whether +# that has taken place, so it runs this function by itself nonetheless, hence +# why we have to turn that into a no-op. +# +# Last but not least, as specified in the top-level comment, we also want to +# support people writing their own Nix(OS)-modules / `jsonify`-esque functions +# for use with the CLI, which this also accomplishes. +if configuration._type or null == "secrets-configuration" then + configuration +else + { + _type = "secrets-configuration"; + + backends.prompt = lib.mapAttrs' (_: backend: { + inherit (backend) name; + value = { + ask = evalDeferredPackage backend.ask; + }; + }) cfg.backends.prompt; + + backends.store = lib.mapAttrs' (_: backend: { + inherit (backend) name; + value = { + get = evalDeferredPackage backend.get; + set = evalDeferredPackage backend.set; + delete = evalDeferredPackage backend.delete; + list = evalDeferredPackage backend.list; + fixup = evalDeferredPackage backend.fixup; + deploy.local = evalDeferredPackage backend.deploy.local; + deploy.remote = evalDeferredPackage backend.deploy.remote; + }; + }) cfg.backends.store; + + store = lib.mapAttrs' (_: secret: { + inherit (secret) name; + value = { + inherit (secret) dependencies backend; + generate = evalDeferredPackage secret.generate; + files = lib.mapAttrs' (_: file: { + inherit (file) name; + value = { + inherit (file) deploy; + }; + }) secret.files; + prompts = lib.mapAttrs' (_: prompt: { + inherit (prompt) name; + value = { + inherit (prompt) + label + description + type + backend + ; + }; + }) secret.prompts; + }; + }) cfg.store; + } diff --git a/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/secrets-config.schema.json b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/secrets-config.schema.json new file mode 100644 index 0000000000000..00e50a04266a0 --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/nixos_secrets/secrets-config.schema.json @@ -0,0 +1,201 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "nix-store-derivation": { + "type": "string", + "pattern": "^/nix/store/.+\\.drv$" + }, + "nullable-nix-store-derivation": { + "oneOf": [ + { + "$ref": "#/$defs/nix-store-derivation" + }, + { + "type": "null" + } + ] + }, + "safe-name": { + "type": "string", + "pattern": "^[a-zA-Z0-9:_\\.-]+$" + } + }, + "title": "SecretsConfiguration", + "description": "A configuration nixos-secrets can consume", + "type": "object", + "required": [ + "_type", + "store", + "backends" + ], + "properties": { + "_type": { + "type": "string", + "description": "A magic string signaling to the CLI that the data needs no further evaluation", + "const": "secrets-configuration" + }, + "store": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/safe-name" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": [ + "backend", + "dependencies", + "prompts", + "files", + "generate" + ], + "properties": { + "backend": { + "type": "string" + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + } + }, + "prompts": { + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": [ + "backend", + "label", + "description", + "type" + ], + "properties": { + "backend": { + "type": "string" + }, + "label": { + "type": "string" + }, + "description": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "hidden", + "line", + "multiline" + ] + } + } + } + }, + "files": { + "propertyNames": { + "$ref": "#/$defs/safe-name" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": [ + "deploy" + ], + "properties": { + "deploy": { + "type": "boolean" + } + } + } + }, + "generate": { + "$ref": "#/$defs/nullable-nix-store-derivation" + } + } + } + }, + "backends": { + "type": "object", + "required": [ + "prompt", + "store" + ], + "properties": { + "prompt": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/safe-name" + }, + "additionalProperties": { + "description": "Derivations for every prompt backend", + "type": "object", + "additionalProperties": false, + "required": [ + "ask" + ], + "properties": { + "ask": { + "$ref": "#/$defs/nix-store-derivation" + } + } + } + }, + "store": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Derivations for every store backend", + "additionalProperties": false, + "required": [ + "get", + "set", + "delete", + "list", + "fixup", + "deploy" + ], + "properties": { + "get": { + "$ref": "#/$defs/nullable-nix-store-derivation" + }, + "set": { + "$ref": "#/$defs/nix-store-derivation" + }, + "delete": { + "$ref": "#/$defs/nullable-nix-store-derivation" + }, + "list": { + "$ref": "#/$defs/nix-store-derivation" + }, + "fixup": { + "$ref": "#/$defs/nullable-nix-store-derivation" + }, + "deploy": { + "type": "object", + "additionalProperties": false, + "required": [ + "local", + "remote" + ], + "properties": { + "local": { + "$ref": "#/$defs/nullable-nix-store-derivation" + }, + "remote": { + "$ref": "#/$defs/nullable-nix-store-derivation" + } + } + } + } + } + } + } + } + } +} diff --git a/pkgs/by-name/ni/nixos-secrets/src/pyproject.toml b/pkgs/by-name/ni/nixos-secrets/src/pyproject.toml new file mode 100644 index 0000000000000..0d3f09265819c --- /dev/null +++ b/pkgs/by-name/ni/nixos-secrets/src/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "nixos-secrets" +version = "0.1.0" + +[project.scripts] +nixos-secrets = "nixos_secrets.cli:main" + +[tool.setuptools.package-data] +nixos_secrets = ["nix/*", "secrets-config.schema.json"]