Skip to content

Commit

Permalink
Merge pull request #11 from cspr-rad/multi-node-nixos-test
Browse files Browse the repository at this point in the history
Multi node nixos test
  • Loading branch information
marijanp authored Apr 25, 2024
2 parents bdda2d5 + 562a4c3 commit b7a3d6d
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 19 deletions.
16 changes: 12 additions & 4 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,21 @@
services.casper-node.package = self.packages.${pkgs.system}.casper-node;
};

checks.x86_64-linux.casper-node-smoke-test =
checks.x86_64-linux =
let
pkgs = nixpkgs.legacyPackages.x86_64-linux;
in
pkgs.callPackage ./nixos/tests/casper-node/smoke-test.nix {
casperNodeModule = self.nixosModules.casper-node;
agenixModule = agenix.nixosModules.age;
{
casper-node-smoke-test =
pkgs.callPackage ./nixos/tests/casper-node/smoke-test.nix {
casperNodeModule = self.nixosModules.casper-node;
agenixModule = agenix.nixosModules.age;
};
casper-node-private-network-test =
pkgs.callPackage ./nixos/tests/casper-node/private-network-test.nix {
casperNodeModule = self.nixosModules.casper-node;
inherit (self.packages.x86_64-linux) casper-client-rs;
};
};
}
(eachDefaultSystem (system:
Expand Down
74 changes: 59 additions & 15 deletions nixos/modules/casper-node.nix
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,23 @@ let
versionsAndHashes = {
"1.5.6" = "sha256-2N2vPKHLKV32RzzZPV004hWH1/lbeZUf3WofTVm+ZZI=";
};
defaultConfigsSrc = pkgs.fetchFromGitHub {
owner = "casper-network";
repo = "casper-protocol-release";
rev = "refs/tags/casper-${cfg.package.version}";
hash = versionsAndHashes.${cfg.package.version};
defaultConfigsSrc = {
"production" =
pkgs.fetchFromGitHub {
owner = "casper-network";
repo = "casper-protocol-release";
rev = "refs/tags/casper-${cfg.package.version}";
hash = versionsAndHashes.${cfg.package.version};
};
"local" = cfg.package.src.outPath;
};
chainspecTomlSrc = {
"production" = "${defaultConfigsSrc."production"}/config/chainspec.toml";
"local" = "${defaultConfigsSrc."local"}/resources/local/chainspec.toml.in";
};
defaultNodeConfigTomlSrc = {
"production" = "${defaultConfigsSrc."production"}/config/config-example.toml";
"local" = "${defaultConfigsSrc."local"}/resources/local/config.toml";
};
in
{
Expand All @@ -29,6 +41,14 @@ in
type = types.package;
};

genesisConfig = mkOption {
type = types.enum [ "production" "local" ];
default = "production";
description = ''
The genesis config type to use.
'';
};

port = mkOption {
type = types.port;
default = 35000;
Expand All @@ -46,6 +66,15 @@ in
'';
};

knownAddresses = mkOption {
type = with types; nullOr (listOf str);
default = null;
example = [ "https://casper.network:0" ];
description = ''
The public addresses of other nodes to connect to in order to join the network.
'';
};

logLevel = mkOption {
type = types.str;
default = "info";
Expand Down Expand Up @@ -97,24 +126,39 @@ in

users.groups.casper-node = { };

environment.etc."casper/${mapDotsToUnderscore cfg.package.version}/chainspec.toml".source = "${defaultConfigsSrc}/config/chainspec.toml";
environment.etc."casper/${mapDotsToUnderscore cfg.package.version}/config.toml" =
environment.etc."casper/${mapDotsToUnderscore cfg.package.version}/chainspec.toml".source =
if cfg.genesisConfig == "production" then chainspecTomlSrc.${cfg.genesisConfig}
else
let
defaultChainspec = builtins.fromTOML (builtins.readFile chainspecTomlSrc."local");
finalChainspec = lib.recursiveUpdate defaultChainspec {
protocol.version = cfg.package.version;
protocol.activation_point = "1970-01-01T01:00:30.000000Z";
};
in
(pkgs.formats.toml { }).generate "config.toml" finalChainspec;

environment.etc."casper/${mapDotsToUnderscore cfg.package.version}/accounts.toml" = mkIf (cfg.genesisConfig == "local") {
source = "${defaultConfigsSrc."local"}/resources/local/accounts.toml";
};

environment.etc."casper/${mapDotsToUnderscore cfg.package.version}/config.toml".source =
let
defaultConfig = builtins.fromTOML (builtins.readFile "${defaultConfigsSrc}/config/config-example.toml");
defaultNodeConfig = builtins.fromTOML (builtins.readFile defaultNodeConfigTomlSrc.${cfg.genesisConfig});

config = lib.recursiveUpdate defaultConfig {
finalNodeConfig = lib.recursiveUpdate defaultNodeConfig {
storage.path = "/var/lib/casper/casper-node"; # TODO get this programatically
diagnostics_port.enabled = false; # TODO this is false in production but true in local
network = {
bind_address = "0.0.0.0:${builtins.toString cfg.port}";
public_address = "${cfg.publicAddress}";
}
// lib.optionalAttrs (!builtins.isNull cfg.knownAddresses) {
known_addresses = cfg.knownAddresses;
};
consensus.secret_key_path = cfg.validatorSecretKeyPath;
};

format = pkgs.formats.toml { };
configTomlFile = format.generate "config.toml" config;
in
{
source = configTomlFile;
};
(pkgs.formats.toml { }).generate "config.toml" finalNodeConfig;
};
}
75 changes: 75 additions & 0 deletions nixos/tests/casper-node/private-network-test.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{ nixosTest
, casperNodeModule
, casper-client-rs
}:
let
port = 35001;
knownAddresses = [
"192.168.1.2:${builtins.toString port}"
"192.168.1.3:${builtins.toString port}"
"192.168.1.4:${builtins.toString port}"
"192.168.1.5:${builtins.toString port}"
"192.168.1.6:${builtins.toString port}"
];

mkCasperNodeServerConfig = { id, publicAddress, knownAddresses }:
{ config, pkgs, lib, ... }: {
imports = [
casperNodeModule
./reset-time.nix
];
services.reset-time.enable = true;
networking.firewall.allowedTCPPorts = [ config.services.casper-node.port 7777 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ address = publicAddress; prefixLength = 24; }];
services.casper-node = {
enable = true;
inherit knownAddresses port;
genesisConfig = "local";
publicAddress = "${publicAddress}:${builtins.toString config.services.casper-node.port}"; # public address assigned by nixos test driver
validatorSecretKeyPath = "${config.services.casper-node.package.src.outPath}/resources/local/secret_keys/node-${builtins.toString id}.pem";
};
};
in
nixosTest {
name = "casper-node private-network test";

nodes = {
server1 = mkCasperNodeServerConfig { id = 1; publicAddress = "192.168.1.2"; inherit knownAddresses; };
server2 = mkCasperNodeServerConfig { id = 2; publicAddress = "192.168.1.3"; inherit knownAddresses; };
server3 = mkCasperNodeServerConfig { id = 3; publicAddress = "192.168.1.4"; inherit knownAddresses; };
server4 = mkCasperNodeServerConfig { id = 4; publicAddress = "192.168.1.5"; inherit knownAddresses; };
server5 = mkCasperNodeServerConfig { id = 5; publicAddress = "192.168.1.6"; inherit knownAddresses; };

client = { config, pkgs, lib, ... }: {
environment.systemPackages = [ casper-client-rs ];
networking.interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ address = "192.168.1.200"; prefixLength = 24; }];
};
};

testScript = ''
import json
start_all()
server1.wait_for_unit("casper-node.service")
server2.wait_for_unit("casper-node.service")
server3.wait_for_unit("casper-node.service")
server4.wait_for_unit("casper-node.service")
server5.wait_for_unit("casper-node.service")
client.wait_for_unit("network-online.target")
server1.wait_for_open_port(7777)
def node_is_validating(node: str):
def _node_is_validating(_) -> bool:
result = json.loads(client.succeed("casper-client get-node-status --node-address http://{}:7777".format(node)))
return result["result"]["reactor_state"] == "Validate"
return _node_is_validating
with client.nested("waiting for nodes to reach validate reactor state"):
retry(node_is_validating("server1"))
retry(node_is_validating("server2"))
retry(node_is_validating("server3"))
retry(node_is_validating("server4"))
# retry(node_is_validating("server5")) # TODO figure out why this one stays in KeepUp
'';
}
32 changes: 32 additions & 0 deletions nixos/tests/casper-node/reset-time.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{ lib, pkgs, config, ... }:
let
inherit (lib)
mkIf
mkMerge
mkEnableOption
;
cfg = config.services.reset-time;
in
{
options.services.reset-time = {

enable = mkEnableOption ("reset-time");

};

config = mkIf cfg.enable {

systemd.services.reset-time =
{
description = "Reset System Time to Epoch 0";
wantedBy = [ "multi-user.target" ];
serviceConfig = mkMerge [
{
ExecStart = "${pkgs.coreutils}/bin/date -s '1970-01-01 01:00:00'";
Type = "oneshot";
}
];
};

};
}

0 comments on commit b7a3d6d

Please sign in to comment.