tests: define tests via flake
Advantages: - Pure test evaluations - The test framework can now be used by flakes that extend nix-bitcoin - Most features of `run-tests.sh` are now accessible via `nix build`/`nix run`. We keep `run-tests.sh` for advanced features like `scenarioOverridesFile` and adhoc scenarios. Other changes: - `run-tests.sh` now builds aggregate VM tests like `basic` or `buildable` by creating all VMs in a single evaluation. This speeds up the tests and eases debugging by separating the eval and build steps. - Use the new `nix` CLI which has improved build output logging by prefixing output lines with the origin drv name.
This commit is contained in:
Erik Arvstedt
@@ -53,7 +53,10 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export containerName=nb-test
|
||||
# These vars are set by ../run-tests.sh
|
||||
: "${container:=}"
|
||||
: "${scriptDir:=}"
|
||||
|
||||
containerCommand=shell
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -69,11 +72,10 @@ while [[ $# -gt 0 ]]; do
|
||||
done
|
||||
|
||||
containerBin=$(type -P extra-container) || true
|
||||
if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then
|
||||
if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.11*) ]]; then
|
||||
echo
|
||||
echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH."
|
||||
nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \
|
||||
-A pinned.extra-container >/dev/null
|
||||
echo "Building extra-container. Skip this step by adding extra-container 0.11 to PATH."
|
||||
nix build --out-link /tmp/extra-container "$scriptDir"/..#extra-container
|
||||
# When this script is run as root, e.g. when run in an extra-container shell,
|
||||
# chown the gcroot symlink to the regular (login) user so that the symlink can be
|
||||
# overwritten when this script is run without root.
|
||||
@@ -83,7 +85,4 @@ if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*)
|
||||
export PATH="/tmp/extra-container/bin${PATH:+:}$PATH"
|
||||
fi
|
||||
|
||||
read -rd '' src <<EOF || true
|
||||
((import "$scriptDir/tests.nix" {}).getTest "$scenario").container
|
||||
EOF
|
||||
exec extra-container "$containerCommand" -E "$src" "$@"
|
||||
exec "$container"/bin/container "$containerCommand" "$@"
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
pkgs:
|
||||
flake: pkgs: makeTestVM:
|
||||
let
|
||||
makeVM = import ./make-test-vm.nix pkgs;
|
||||
inherit (pkgs) lib;
|
||||
inherit (flake.inputs) extra-container;
|
||||
inherit (pkgs.stdenv.hostPlatform) system;
|
||||
in
|
||||
|
||||
name: testConfig:
|
||||
{
|
||||
vm = makeVM {
|
||||
name = "nix-bitcoin-${name}";
|
||||
{ name ? "nix-bitcoin-test", config }:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
|
||||
testConfig = config;
|
||||
|
||||
test = makeTestVM {
|
||||
inherit name;
|
||||
|
||||
nodes.machine = { config, ... }: {
|
||||
imports = [ testConfig ];
|
||||
|
||||
virtualisation = {
|
||||
# Needed because duplicity requires 270 MB of free temp space, regardless of backup size
|
||||
diskSize = 1024;
|
||||
|
||||
# Min. 800 MiB needed to avoid 'out of memory' errors
|
||||
memorySize = lib.mkDefault 2048;
|
||||
|
||||
cores = lib.mkDefault 2;
|
||||
};
|
||||
imports = [
|
||||
testConfig
|
||||
commonVmConfig
|
||||
];
|
||||
|
||||
test.shellcheckServices.enable = true;
|
||||
};
|
||||
@@ -47,7 +44,7 @@ name: testConfig:
|
||||
(builtins.readFile ./../tests.py)
|
||||
cfg.test.extraTestScript
|
||||
# Don't run tests in interactive mode.
|
||||
# is_interactive is set in ../run-tests.sh
|
||||
# is_interactive is set in ./run-vm.sh
|
||||
''
|
||||
if not "is_interactive" in vars():
|
||||
run_tests()
|
||||
@@ -55,40 +52,53 @@ name: testConfig:
|
||||
];
|
||||
};
|
||||
|
||||
container = {
|
||||
# The container name has a 11 char length limit
|
||||
containers.nb-test = { config, ... }: {
|
||||
imports = [
|
||||
{
|
||||
config = {
|
||||
extra = config.config.test.container;
|
||||
config = testConfig;
|
||||
};
|
||||
}
|
||||
# A VM runner for interactive use
|
||||
run = pkgs.writers.writeBashBin "run-vm" ''
|
||||
. ${./run-vm.sh} ${test.driver} "$@"
|
||||
'';
|
||||
|
||||
# Enable FUSE inside the container when clightning replication
|
||||
# is enabled.
|
||||
# TODO-EXTERNAL: Remove this when
|
||||
# https://github.com/systemd/systemd/issues/17607
|
||||
# has been resolved. This will also improve security.
|
||||
(
|
||||
let
|
||||
s = config.config.services;
|
||||
in
|
||||
lib.mkIf (s ? clightning && s.clightning.enable && s.clightning.replication.enable) {
|
||||
bindMounts."/dev/fuse" = { hostPath = "/dev/fuse"; };
|
||||
allowedDevices = [ { node = "/dev/fuse"; modifier = "rw"; } ];
|
||||
mkContainer = legacyInstallDirs:
|
||||
extra-container.lib.buildContainers {
|
||||
inherit system legacyInstallDirs;
|
||||
config = {
|
||||
# The container name has a 11 char length limit
|
||||
containers.nb-test = { config, ... }: {
|
||||
imports = [
|
||||
{
|
||||
config = {
|
||||
extra = config.config.test.container;
|
||||
config = testConfig;
|
||||
};
|
||||
}
|
||||
)
|
||||
];
|
||||
|
||||
# Enable FUSE inside the container when clightning replication
|
||||
# is enabled.
|
||||
# TODO-EXTERNAL: Remove this when
|
||||
# https://github.com/systemd/systemd/issues/17607
|
||||
# has been resolved. This will also improve security.
|
||||
(
|
||||
let
|
||||
s = config.config.services;
|
||||
in
|
||||
lib.mkIf (s ? clightning && s.clightning.enable && s.clightning.replication.enable) {
|
||||
bindMounts."/dev/fuse" = { hostPath = "/dev/fuse"; };
|
||||
allowedDevices = [ { node = "/dev/fuse"; modifier = "rw"; } ];
|
||||
}
|
||||
)
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
container = mkContainer false;
|
||||
containerLegacy = mkContainer true;
|
||||
|
||||
# This allows running a test scenario in a regular NixOS VM.
|
||||
# No tests are executed.
|
||||
vmWithoutTests = (pkgs.nixos ({ config, ... }: {
|
||||
vm = (pkgs.nixos ({ config, ... }: {
|
||||
imports = [
|
||||
testConfig
|
||||
commonVmConfig
|
||||
(pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix")
|
||||
];
|
||||
virtualisation.graphics = false;
|
||||
@@ -103,7 +113,37 @@ name: testConfig:
|
||||
'';
|
||||
|
||||
system.stateVersion = lib.mkDefault config.system.nixos.release;
|
||||
})).config.system.build.vm;
|
||||
})).config.system.build.vm.overrideAttrs (old: {
|
||||
meta = old.meta // { mainProgram = "run-vm-in-tmpdir"; };
|
||||
buildCommand = old.buildCommand + "\n" + ''
|
||||
install -m 700 ${./run-vm-without-tests.sh} $out/bin/run-vm-in-tmpdir
|
||||
patchShebangs $out/bin/run-vm-in-tmpdir
|
||||
'';
|
||||
});
|
||||
|
||||
commonVmConfig = {
|
||||
virtualisation = {
|
||||
# Needed because duplicity requires 270 MB of free temp space, regardless of backup size
|
||||
diskSize = 1024;
|
||||
|
||||
# Min. 800 MiB needed to avoid 'out of memory' errors
|
||||
memorySize = lib.mkDefault 2048;
|
||||
|
||||
# There are no perf gains beyond 3 cores.
|
||||
# Benchmark: Ryzen 7 2700 (8 cores), VM test `default` as of 34f6eb90.
|
||||
# Num. Cores | 1 | 2 | 3 | 4 | 6
|
||||
# Runtime (sec) | 125 | 95 | 89 | 89 | 90
|
||||
cores = lib.mkDefault 3;
|
||||
};
|
||||
};
|
||||
in
|
||||
test // {
|
||||
inherit
|
||||
run
|
||||
vm
|
||||
container
|
||||
# For NixOS with `system.stateVersion` <22.05
|
||||
containerLegacy;
|
||||
|
||||
config = testConfig;
|
||||
}
|
||||
|
||||
30
test/lib/run-vm-without-tests.sh
Normal file
30
test/lib/run-vm-without-tests.sh
Normal file
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# This script uses the following env vars:
|
||||
# NIX_BITCOIN_VM_ENABLE_NETWORK
|
||||
# NIX_BITCOIN_VM_DATADIR
|
||||
# QEMU_OPTS
|
||||
# QEMU_NET_OPTS
|
||||
|
||||
if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
|
||||
dataDir=$NIX_BITCOIN_VM_DATADIR
|
||||
else
|
||||
dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
|
||||
trap 'rm -rf "$dataDir"' EXIT
|
||||
fi
|
||||
|
||||
if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
|
||||
QEMU_NET_OPTS='restrict=on'
|
||||
fi
|
||||
|
||||
# TODO-EXTERNAL:
|
||||
# Pass PATH because run-*-vm is impure (requires coreutils from PATH)
|
||||
env -i \
|
||||
PATH="$PATH" \
|
||||
USE_TMPDIR=1 \
|
||||
TMPDIR="$dataDir" \
|
||||
NIX_DISK_IMAGE="$dataDir/img.qcow2" \
|
||||
QEMU_OPTS="${QEMU_OPTS:-}" \
|
||||
QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
|
||||
"${BASH_SOURCE[0]%/*}"/run-*-vm
|
||||
50
test/lib/run-vm.sh
Normal file
50
test/lib/run-vm.sh
Normal file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# This script uses the following env vars:
|
||||
# NIX_BITCOIN_VM_ENABLE_NETWORK
|
||||
# NIX_BITCOIN_VM_DATADIR
|
||||
# QEMU_OPTS
|
||||
# QEMU_NET_OPTS
|
||||
|
||||
if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
|
||||
dataDir=$NIX_BITCOIN_VM_DATADIR
|
||||
else
|
||||
dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
|
||||
trap 'rm -rf "$dataDir"' EXIT
|
||||
fi
|
||||
|
||||
testDriver=$1
|
||||
shift
|
||||
|
||||
# Variable 'tests' contains the Python code that is executed by the driver on startup
|
||||
if [[ ${1:-} == --debug ]]; then
|
||||
shift
|
||||
echo "Running interactive testing environment"
|
||||
# Start REPL.
|
||||
# Use `code.interact` for the REPL instead of the builtin test driver REPL
|
||||
# because it supports low featured terminals like Emacs' shell-mode.
|
||||
tests='
|
||||
is_interactive = True
|
||||
exec(open(os.environ["testScript"]).read())
|
||||
if "machine" in vars(): machine.start()
|
||||
import code
|
||||
code.interact(local=globals())
|
||||
'
|
||||
echo
|
||||
echo "Starting VM, data dir: $dataDir"
|
||||
else
|
||||
tests='exec(open(os.environ["testScript"]).read())'
|
||||
fi
|
||||
|
||||
if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
|
||||
QEMU_NET_OPTS='restrict=on'
|
||||
fi
|
||||
|
||||
# The VM creates a VDE control socket in $PWD
|
||||
env --chdir "$dataDir" -i \
|
||||
USE_TMPDIR=1 \
|
||||
TMPDIR="$dataDir" \
|
||||
QEMU_OPTS="-nographic ${QEMU_OPTS:-}" \
|
||||
QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
|
||||
"$testDriver/bin/nixos-test-driver" <(echo "$tests") "$@"
|
||||
Reference in New Issue
Block a user