From b5e81d5c0dd51a7e61d0e4561948e33c2414d532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Sun, 15 Sep 2024 21:10:12 +0100 Subject: [PATCH 1/6] syncthing: handle encryptionPassword secret Rewrite the syncthing config update script to embed secrets into the json request. Specifically, we handle the `encryptionPassword` secret. With this code, the user can embed path to the encrpyption password for a given device the folder is shared with, and have it loaded in, without touching the nix store. --- .../modules/services/networking/syncthing.nix | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix index 94ff838b50e04..e6bdb3679edef 100644 --- a/nixos/modules/services/networking/syncthing.nix +++ b/nixos/modules/services/networking/syncthing.nix @@ -103,9 +103,66 @@ let # don't exist in the array given. That's why we use here `POST`, and # only if s.override == true then we DELETE the relevant folders # afterwards. - (map (new_cfg: '' - curl -d ${lib.escapeShellArg (builtins.toJSON new_cfg)} -X POST ${s.baseAddress} - '')) + (map (new_cfg: + let + isSecret = attr: value: builtins.isString value && attr == "encryptionPassword"; + + resolveSecrets = attr: value: + if builtins.isAttrs value then + # Attribute set: process each attribute + builtins.mapAttrs (name: val: resolveSecrets name val) value + else if builtins.isList value then + # List: process each element + map (item: resolveSecrets "" item) value + else if isSecret attr value then + # String that looks like a path: replace with placeholder + let + varName = "secret_${builtins.hashString "sha256" value}"; + in + "\${${varName}}" + else + # Other types: return as is + value; + + # Function to collect all file paths from the configuration + collectPaths = attr: value: + if builtins.isAttrs value then + concatMap (name: collectPaths name value.${name}) (builtins.attrNames value) + else if builtins.isList value then + concatMap (name: collectPaths "" name) value + else if isSecret attr value then + [ value ] + else + []; + + # Function to generate variable assignments for the secrets + generateSecretVars = paths: + concatStringsSep "\n" (map (path: + let + varName = "secret_${builtins.hashString "sha256" path}"; + in + '' + if [ ! -r ${path} ]; then + echo "${path} does not exist" + exit 1 + fi + ${varName}=$(<${path}) + '' + ) paths); + + resolved_cfg = resolveSecrets "" new_cfg; + secretPaths = collectPaths "" new_cfg; + secretVarsScript = generateSecretVars secretPaths; + + jsonString = builtins.toJSON resolved_cfg; + escapedJson = builtins.replaceStrings ["\""] ["\\\""] jsonString; + in + '' + ${secretVarsScript} + + curl -d "${escapedJson}" -X POST ${s.baseAddress} + '' + )) (lib.concatStringsSep "\n") ] /* If we need to override devices/folders, we iterate all currently configured From 0bc8c16f7da33f03bbf092689ff697828b35c3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Sun, 15 Sep 2024 21:10:12 +0100 Subject: [PATCH 2/6] syncthing: expose encryptionPassword - Change `folder.devices` type into `oneOf [(listOf str) (attrsOf (submodule { ... }))]`. - Expose `encryptionPassord` within the attrSet of the devices option. This allows the user to set the encrpyption password use to share the folder's data with. We do this by file path, as opposed to string literal, because we do not want to embed the encrpyption password into the nix store. --- .../modules/services/networking/syncthing.nix | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix index e6bdb3679edef..7995238ab8d47 100644 --- a/nixos/modules/services/networking/syncthing.nix +++ b/nixos/modules/services/networking/syncthing.nix @@ -34,12 +34,22 @@ let The options services.syncthing.settings.folders..{rescanInterval,watch,watchDelay} were removed. Please use, respectively, {rescanIntervalS,fsWatcherEnabled,fsWatcherDelayS} instead. '' { - devices = map (device: - if builtins.isString device then - { deviceId = cfg.settings.devices.${device}.id; } + devices = let + folderDevices = folder.devices; + in + if builtins.isList folderDevices then + map (device: + if builtins.isString device then + { deviceId = cfg.settings.devices.${device}.id; } + else + device + ) folderDevices + else if builtins.isAttrs folderDevices then + mapAttrsToList (deviceName: deviceValue: + deviceValue // { deviceId = cfg.settings.devices.${deviceName}.id; } + ) folderDevices else - device - ) folder.devices; + throw "Invalid type for devices in folder '${folderName}'; expected list or attrset."; }) (filterAttrs (_: folder: folder.enable ) cfg.settings.folders); @@ -435,11 +445,30 @@ in { }; devices = mkOption { - type = types.listOf types.str; + type = types.oneOf [ + (types.listOf types.str) + (types.attrsOf (types.submodule ({ name, ... }: { + freeformType = settingsFormat.type; + options = { + encryptionPassword = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Path to encryption password. If set, the file will be read during + service activation, without being embedded in derivation. + ''; + }; + }; + })) + )]; default = []; description = '' The devices this folder should be shared with. Each device must be defined in the [devices](#opt-services.syncthing.settings.devices) option. + + Either a list of strings, or an attribute set, where keys are defined in the + [devices](#opt-services.syncthing.settings.devices) option, and values are + device configurations. ''; }; From 3dcbfdcade5d64ea83054b0bba9afa4303b58610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Thu, 19 Sep 2024 16:31:20 +0100 Subject: [PATCH 3/6] nixosTests.syncthing: move tests to their own directory --- nixos/tests/all-tests.nix | 8 ++++---- nixos/tests/{syncthing.nix => syncthing/default.nix} | 2 +- nixos/tests/{syncthing-init.nix => syncthing/init.nix} | 2 +- .../many-devices.nix} | 2 +- .../no-settings.nix} | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename nixos/tests/{syncthing.nix => syncthing/default.nix} (97%) rename nixos/tests/{syncthing-init.nix => syncthing/init.nix} (92%) rename nixos/tests/{syncthing-many-devices.nix => syncthing/many-devices.nix} (99%) rename nixos/tests/{syncthing-no-settings.nix => syncthing/no-settings.nix} (89%) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 6c9ff9fb9c200..9787f2814ed43 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -941,10 +941,10 @@ in { switchTestNg = handleTest ./switch-test.nix { ng = true; }; sx = handleTest ./sx.nix {}; sympa = handleTest ./sympa.nix {}; - syncthing = handleTest ./syncthing.nix {}; - syncthing-no-settings = handleTest ./syncthing-no-settings.nix {}; - syncthing-init = handleTest ./syncthing-init.nix {}; - syncthing-many-devices = handleTest ./syncthing-many-devices.nix {}; + syncthing = handleTest ./syncthing/default.nix {}; + syncthing-no-settings = handleTest ./syncthing/no-settings.nix {}; + syncthing-init = handleTest ./syncthing/init.nix {}; + syncthing-many-devices = handleTest ./syncthing/many-devices.nix {}; syncthing-relay = handleTest ./syncthing-relay.nix {}; sysinit-reactivation = runTest ./sysinit-reactivation.nix; systemd = handleTest ./systemd.nix {}; diff --git a/nixos/tests/syncthing.nix b/nixos/tests/syncthing/default.nix similarity index 97% rename from nixos/tests/syncthing.nix rename to nixos/tests/syncthing/default.nix index aff1d87441308..996ceb9c24c7c 100644 --- a/nixos/tests/syncthing.nix +++ b/nixos/tests/syncthing/default.nix @@ -1,4 +1,4 @@ -import ./make-test-python.nix ({ lib, pkgs, ... }: { +import ../make-test-python.nix ({ lib, pkgs, ... }: { name = "syncthing"; meta.maintainers = with pkgs.lib.maintainers; [ chkno ]; diff --git a/nixos/tests/syncthing-init.nix b/nixos/tests/syncthing/init.nix similarity index 92% rename from nixos/tests/syncthing-init.nix rename to nixos/tests/syncthing/init.nix index 97fcf2ad28d10..234c556546033 100644 --- a/nixos/tests/syncthing-init.nix +++ b/nixos/tests/syncthing/init.nix @@ -1,4 +1,4 @@ -import ./make-test-python.nix ({ lib, pkgs, ... }: let +import ../make-test-python.nix ({ lib, pkgs, ... }: let testId = "7CFNTQM-IMTJBHJ-3UWRDIU-ZGQJFR6-VCXZ3NB-XUH3KZO-N52ITXR-LAIYUAU"; diff --git a/nixos/tests/syncthing-many-devices.nix b/nixos/tests/syncthing/many-devices.nix similarity index 99% rename from nixos/tests/syncthing-many-devices.nix rename to nixos/tests/syncthing/many-devices.nix index 2251bf0774533..9090a52af8a00 100644 --- a/nixos/tests/syncthing-many-devices.nix +++ b/nixos/tests/syncthing/many-devices.nix @@ -1,4 +1,4 @@ -import ./make-test-python.nix ({ lib, pkgs, ... }: +import ../make-test-python.nix ({ lib, pkgs, ... }: # This nixosTest is supposed to check the following: # diff --git a/nixos/tests/syncthing-no-settings.nix b/nixos/tests/syncthing/no-settings.nix similarity index 89% rename from nixos/tests/syncthing-no-settings.nix rename to nixos/tests/syncthing/no-settings.nix index fee122b5e35c0..addb033a508a2 100644 --- a/nixos/tests/syncthing-no-settings.nix +++ b/nixos/tests/syncthing/no-settings.nix @@ -1,4 +1,4 @@ -import ./make-test-python.nix ({ lib, pkgs, ... }: { +import ../make-test-python.nix ({ lib, pkgs, ... }: { name = "syncthing"; meta.maintainers = with pkgs.lib.maintainers; [ chkno ]; From 137b008b5e5b31c08cbc56e15ae86cdf5880393d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Thu, 19 Sep 2024 16:36:23 +0100 Subject: [PATCH 4/6] nixosTests.syncthing: define test for declarative folders --- nixos/tests/all-tests.nix | 1 + nixos/tests/syncthing/folders.nix | 69 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 nixos/tests/syncthing/folders.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 9787f2814ed43..7ce5818b50ef3 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -945,6 +945,7 @@ in { syncthing-no-settings = handleTest ./syncthing/no-settings.nix {}; syncthing-init = handleTest ./syncthing/init.nix {}; syncthing-many-devices = handleTest ./syncthing/many-devices.nix {}; + syncthing-folders = handleTest ./syncthing/folders.nix {}; syncthing-relay = handleTest ./syncthing-relay.nix {}; sysinit-reactivation = runTest ./sysinit-reactivation.nix; systemd = handleTest ./systemd.nix {}; diff --git a/nixos/tests/syncthing/folders.nix b/nixos/tests/syncthing/folders.nix new file mode 100644 index 0000000000000..5914aacf0b2a3 --- /dev/null +++ b/nixos/tests/syncthing/folders.nix @@ -0,0 +1,69 @@ +import ../make-test-python.nix ( + { lib, pkgs, ... }: + let + genNodeId = + name: + pkgs.runCommand "syncthing-test-certs-${name}" { } '' + mkdir -p $out + ${pkgs.syncthing}/bin/syncthing generate --config=$out + ${pkgs.libxml2}/bin/xmllint --xpath 'string(configuration/device/@id)' $out/config.xml > $out/id + ''; + idA = genNodeId "a"; + idB = genNodeId "b"; + in + { + name = "syncthing"; + meta.maintainers = with pkgs.lib.maintainers; [ zarelit ]; + + nodes = { + a = { + services.syncthing = { + enable = true; + openDefaultPorts = true; + cert = "${idA}/cert.pem"; + key = "${idA}/key.pem"; + settings = { + devices.b = { + id = lib.fileContents "${idB}/id"; + }; + folders.foo = { + path = "/var/lib/syncthing/foo"; + devices = [ "b" ]; + }; + }; + }; + }; + b = { + services.syncthing = { + enable = true; + openDefaultPorts = true; + cert = "${idB}/cert.pem"; + key = "${idB}/key.pem"; + settings = { + devices.a = { + id = lib.fileContents "${idA}/id"; + }; + folders.foo = { + path = "/var/lib/syncthing/foo"; + devices = [ "a" ]; + }; + }; + }; + }; + }; + + testScript = '' + start_all() + a.wait_for_unit("syncthing.service") + b.wait_for_unit("syncthing.service") + a.wait_for_open_port(22000) + b.wait_for_open_port(22000) + a.wait_for_file("/var/lib/syncthing/foo") + b.wait_for_file("/var/lib/syncthing/foo") + a.succeed("echo a2b > /var/lib/syncthing/foo/a2b") + b.succeed("echo b2a > /var/lib/syncthing/foo/b2a") + a.wait_for_file("/var/lib/syncthing/foo/b2a") + b.wait_for_file("/var/lib/syncthing/foo/a2b") + ''; + } +) From 3c04dffb094bc836931f889c31ff15f1281ccab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Thu, 19 Sep 2024 16:51:11 +0100 Subject: [PATCH 5/6] nixosTests.syncthing: create encrypted device test --- nixos/tests/syncthing/folders.nix | 63 ++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/nixos/tests/syncthing/folders.nix b/nixos/tests/syncthing/folders.nix index 5914aacf0b2a3..46985b4709a69 100644 --- a/nixos/tests/syncthing/folders.nix +++ b/nixos/tests/syncthing/folders.nix @@ -10,6 +10,8 @@ import ../make-test-python.nix ( ''; idA = genNodeId "a"; idB = genNodeId "b"; + idC = genNodeId "c"; + testPasswordFile = pkgs.writeText "syncthing-test-password" "it's a secret"; in { name = "syncthing"; @@ -23,13 +25,16 @@ import ../make-test-python.nix ( cert = "${idA}/cert.pem"; key = "${idA}/key.pem"; settings = { - devices.b = { - id = lib.fileContents "${idB}/id"; - }; + devices.b.id = lib.fileContents "${idB}/id"; + devices.c.id = lib.fileContents "${idC}/id"; folders.foo = { path = "/var/lib/syncthing/foo"; devices = [ "b" ]; }; + folders.bar = { + path = "/var/lib/syncthing/bar"; + devices.c.encryptionPassword = "${testPasswordFile}"; + }; }; }; }; @@ -40,13 +45,36 @@ import ../make-test-python.nix ( cert = "${idB}/cert.pem"; key = "${idB}/key.pem"; settings = { - devices.a = { - id = lib.fileContents "${idA}/id"; - }; + devices.a.id = lib.fileContents "${idA}/id"; + devices.c.id = lib.fileContents "${idC}/id"; folders.foo = { path = "/var/lib/syncthing/foo"; devices = [ "a" ]; }; + folders.bar = { + path = "/var/lib/syncthing/bar"; + devices.c.encryptionPassword = "${testPasswordFile}"; + }; + }; + }; + }; + c = { + services.syncthing = { + enable = true; + openDefaultPorts = true; + cert = "${idC}/cert.pem"; + key = "${idC}/key.pem"; + settings = { + devices.a.id = lib.fileContents "${idA}/id"; + devices.b.id = lib.fileContents "${idB}/id"; + folders.bar = { + path = "/var/lib/syncthing/bar"; + devices = [ + "a" + "b" + ]; + type = "receiveencrypted"; + }; }; }; }; @@ -54,16 +82,39 @@ import ../make-test-python.nix ( testScript = '' start_all() + a.wait_for_unit("syncthing.service") b.wait_for_unit("syncthing.service") + c.wait_for_unit("syncthing.service") a.wait_for_open_port(22000) b.wait_for_open_port(22000) + c.wait_for_open_port(22000) + + # Test foo + a.wait_for_file("/var/lib/syncthing/foo") b.wait_for_file("/var/lib/syncthing/foo") + a.succeed("echo a2b > /var/lib/syncthing/foo/a2b") b.succeed("echo b2a > /var/lib/syncthing/foo/b2a") + a.wait_for_file("/var/lib/syncthing/foo/b2a") b.wait_for_file("/var/lib/syncthing/foo/a2b") + + # Test bar + + a.wait_for_file("/var/lib/syncthing/bar") + b.wait_for_file("/var/lib/syncthing/bar") + c.wait_for_file("/var/lib/syncthing/bar") + + a.succeed("echo plaincontent > /var/lib/syncthing/bar/plainname") + + # B should be able to decrypt, check that content of file matches + b.wait_for_file("/var/lib/syncthing/bar/plainname") + b.succeed("grep plaincontent /var/lib/syncthing/bar/plainname") + + # Bar on C is untrusted, check that content is not in cleartext + c.fail("grep -R plaincontent /var/lib/syncthing/bar") ''; } ) From 3018b7de6a5e864f6cb4d1395387129ae81b5d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Mon, 30 Sep 2024 10:46:00 +0100 Subject: [PATCH 6/6] doc: 24.11: release note for syncthing changes --- nixos/doc/manual/release-notes/rl-2411.section.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index e609253950f14..d5e279bfd536e 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -518,6 +518,8 @@ - `nix.channel.enable = false` no longer implies `nix.settings.nix-path = []`. Since Nix 2.13, a `nix-path` set in `nix.conf` cannot be overriden by the `NIX_PATH` configuration variable. +- `services.syncthing` now accepts an `attrSet` for `folders[].devices`, allowing to set `encryptionPassword` file for a device. + ## Detailed migration information {#sec-release-24.11-migration} ### `sound` options removal {#sec-release-24.11-migration-sound}