From 09ef1329f1d41eb51d3ec8f37dd859078e24270d Mon Sep 17 00:00:00 2001
From: "Alexander V. Nikolaev" <avn@avnik.info>
Date: Thu, 9 Nov 2023 13:13:58 +0200
Subject: [PATCH 1/7] Dummy test

---
 tests/cardano-cli.nix | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 tests/cardano-cli.nix

diff --git a/tests/cardano-cli.nix b/tests/cardano-cli.nix
new file mode 100644
index 0000000..4e4c6f8
--- /dev/null
+++ b/tests/cardano-cli.nix
@@ -0,0 +1,26 @@
+_: {
+  cardanoNix.tests = {
+    dummy = {
+      systems = ["x86_64-linux"];
+
+      module = {
+        name = "cli-test";
+
+        nodes = {
+          machine = {
+            virtualisation = {
+              cores = 2;
+              memorySize = 1024;
+              writableStore = true;
+            };
+            cardano-ecosystem.cli.enable = true;
+          };
+        };
+        testScript = ''
+          # FIXME: check for cardano cli, not git
+          machine.succeed("git --version")
+        '';
+      };
+    };
+  };
+}

From 55a5ac70d2196e6f8b438251f61b8325811054ab Mon Sep 17 00:00:00 2001
From: "Alexander V. Nikolaev" <avn@avnik.info>
Date: Thu, 9 Nov 2023 17:02:27 +0200
Subject: [PATCH 2/7] feat: Prototype of tests

---
 checks/default.nix |  39 +++++++-----
 checks/testing.nix | 152 +++++++++++++++++++++++++++++++++++++++++++++
 flake.nix          |   1 +
 lib/default.nix    |   5 +-
 tests/default.nix  |   5 ++
 5 files changed, 186 insertions(+), 16 deletions(-)
 create mode 100644 checks/testing.nix
 create mode 100644 tests/default.nix

diff --git a/checks/default.nix b/checks/default.nix
index 7e198b6..8efee61 100644
--- a/checks/default.nix
+++ b/checks/default.nix
@@ -1,24 +1,33 @@
-{inputs, ...}: {
+{
+  inputs,
+  self,
+  ...
+}: {
+  imports = [
+    ./testing.nix
+  ];
   perSystem = {
     pkgs,
     config,
     ...
   }: {
-    apps = {
-      nix-build-all.program = pkgs.writeShellApplication {
-        name = "nix-build-all";
-        runtimeInputs = [
-          (pkgs.callPackage inputs.devour-flake {})
-        ];
-        text = ''
-          # Make sure that flake.lock is sync
-          nix flake lock --no-update-lock-file
+    apps =
+      {
+        nix-build-all.program = pkgs.writeShellApplication {
+          name = "nix-build-all";
+          runtimeInputs = [
+            (pkgs.callPackage inputs.devour-flake {})
+          ];
+          text = ''
+            # Make sure that flake.lock is sync
+            nix flake lock --no-update-lock-file
 
-          # Do a full nix build (all outputs)
-          devour-flake . "$@"
-        '';
-      };
-    };
+            # Do a full nix build (all outputs)
+            devour-flake . "$@"
+          '';
+        };
+      }
+      // self.cardanoNix.checks;
 
     devshells.default.commands = [
       {
diff --git a/checks/testing.nix b/checks/testing.nix
new file mode 100644
index 0000000..130b2e7
--- /dev/null
+++ b/checks/testing.nix
@@ -0,0 +1,152 @@
+{
+  self,
+  inputs,
+  ...
+}:
+with self.lib; {
+  perSystem = {
+    lib,
+    config,
+    system,
+    ...
+  }: let
+    # create a custom nixpkgs with our flake packages available
+    pkgs = import inputs.nixpkgs {
+      inherit system;
+      overlays = [
+      ];
+    };
+  in {
+    ########################################
+    ## Interface
+    ########################################
+    options.cardanoNix = with lib; {
+      checks = mkOption {
+        type = types.attrsOf types.package;
+        default = {};
+        internal = true;
+      };
+      tests = mkOption {
+        type = types.lazyAttrsOf (types.submodule {
+          options = {
+            systems = mkOption {
+              type = types.listOf types.str;
+            };
+            module = mkOption {
+              type = types.defferedModule;
+            };
+          };
+        });
+      };
+    };
+
+    ########################################
+    ## Implementation
+    ########################################
+    config.cardanoNix.checks = with lib; let
+      # import the testing framework
+      nixos-lib = import (pkgs.path + "/nixos/lib") {};
+
+      # examine the `systems` attribute of each test, filtering out any that do not support the current system
+      eachTestForSystem = with lib;
+        filterAttrs
+        (_: v: elem system v.systems)
+        config.cardanoNix.tests;
+    in
+      mapAttrs'
+      (name: test:
+        nameValuePair "testing-${removeSuffix ".test" name}"
+        (nixos-lib.runTest {
+          hostPkgs = pkgs;
+
+          # speed up evaluation by skipping docs
+          defaults.documentation.enable = lib.mkDefault false;
+
+          # make self available in test modules and our custom pkgs
+          node.specialArgs = {inherit self pkgs;};
+
+          # import all of our flake nixos modules by default
+          defaults.imports = [
+            self.nixosModules.default
+          ];
+
+          # import the test module
+          imports = [test.module];
+        })
+        .config
+        .result)
+      eachTestForSystem;
+
+    ########################################
+    ## Commands
+    ########################################
+    config.devshells.default.commands = [
+      {
+        name = "tests";
+        category = "Testing";
+        help = "Build and run a test";
+        command = with lib; ''
+          Help() {
+               # Display Help
+               echo "  Build and run a test"
+               echo
+               echo "  Usage:"
+               echo "    test <name>"
+               echo "    test <name> --interactive"
+               echo "    test -s <system> <name>"
+               echo
+               echo "  Arguments:"
+               echo "    <name> If a test package is called 'testing-nethermind-basic' then <name> should be 'nethermind-basic'."
+               echo
+               echo "  Options:"
+               echo "    -h --help          Show this screen."
+               echo "    -l --list          Show available tests."
+               echo "    -s --system        Specify the target platform [default: x84_64-linux]."
+               echo "    -i --interactive   Run the test interactively."
+               echo
+          }
+
+          List() {
+            # Display available tests
+            echo "  List of available tests:"
+            echo
+            echo "${strings.concatMapStrings (s: "    - " + s + "\n") (attrsets.mapAttrsToList (name: _: (removePrefix "testing-" name)) config.cardanoNix.checks)}"
+          }
+
+          ARGS=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
+          eval set -- "$ARGS"
+
+          SYSTEM="x86_64-linux"
+          DRIVER_ARGS=()
+
+          while [ $# -gt 0 ]; do
+            case "$1" in
+                -i | --interactive) DRIVER_ARGS+=("--interactive"); shift;;
+                -s | --system) SYSTEM="$2"; shift 2;;
+                -h | --help) Help; exit 0;;
+                -l | --list) List; exit 0;;
+                -- ) shift; break;;
+                * ) break;;
+            esac
+          done
+
+          if [ $# -eq 0 ]; then
+            # No test name has been provided
+            Help
+            exit 1
+          fi
+
+          NAME="$1"
+          shift
+
+          # build the test driver
+          DRIVER=$(nix build ".#checks.$SYSTEM.testing-$NAME.driver" --print-out-paths --no-link)
+
+          # run the test driver, passing any remaining arguments
+          set -x
+          ''${DRIVER}/bin/nixos-test-driver "''${DRIVER_ARGS[@]}"
+        '';
+      }
+    ];
+  };
+}
diff --git a/flake.nix b/flake.nix
index 05cb35c..acd32cc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -33,6 +33,7 @@
         ./formatter
         ./modules
         ./shell
+        ./tests
       ];
       systems = [
         "x86_64-linux"
diff --git a/lib/default.nix b/lib/default.nix
index c8d977e..cc1bbec 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,5 +1,8 @@
 {lib, ...}: let
   lib' = import ./functions.nix lib;
 in {
-  flake.lib = lib';
+  options.cardanoNix = lib.mkOption {
+    type = lib.types.lazyAttrsOf lib.types.submodule;
+  };
+  config.flake.lib = lib';
 }
diff --git a/tests/default.nix b/tests/default.nix
new file mode 100644
index 0000000..a943cfe
--- /dev/null
+++ b/tests/default.nix
@@ -0,0 +1,5 @@
+{
+  imports = [
+    ./cardano-cli.nix
+  ];
+}

From bbe2c88461b6ef42c677ec0ed4f0543c5c5a0ad5 Mon Sep 17 00:00:00 2001
From: Andrea Ciceri <andrea.ciceri@autistici.org>
Date: Fri, 10 Nov 2023 12:52:07 +0100
Subject: [PATCH 3/7] fix: partial WIP

---
 checks/default.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/checks/default.nix b/checks/default.nix
index 8efee61..e7f0311 100644
--- a/checks/default.nix
+++ b/checks/default.nix
@@ -1,6 +1,5 @@
 {
   inputs,
-  self,
   ...
 }: {
   imports = [
@@ -27,7 +26,7 @@
           '';
         };
       }
-      // self.cardanoNix.checks;
+      // config.cardanoNix.checks;
 
     devshells.default.commands = [
       {

From cb8a06460cdac4a306cccb86c450e5cc0c00e7ec Mon Sep 17 00:00:00 2001
From: Andrea Ciceri <andrea.ciceri@autistici.org>
Date: Fri, 10 Nov 2023 18:24:14 +0100
Subject: [PATCH 4/7] fix: testing

---
 checks/default.nix  |  35 +++++------
 checks/testing.nix  | 141 +++++++++++++++++++++-----------------------
 flake.nix           |   4 +-
 lib/default.nix     |   3 -
 modules/default.nix |   2 +-
 5 files changed, 85 insertions(+), 100 deletions(-)

diff --git a/checks/default.nix b/checks/default.nix
index e7f0311..b0b2e84 100644
--- a/checks/default.nix
+++ b/checks/default.nix
@@ -1,7 +1,4 @@
-{
-  inputs,
-  ...
-}: {
+{inputs, ...}: {
   imports = [
     ./testing.nix
   ];
@@ -10,23 +7,21 @@
     config,
     ...
   }: {
-    apps =
-      {
-        nix-build-all.program = pkgs.writeShellApplication {
-          name = "nix-build-all";
-          runtimeInputs = [
-            (pkgs.callPackage inputs.devour-flake {})
-          ];
-          text = ''
-            # Make sure that flake.lock is sync
-            nix flake lock --no-update-lock-file
+    apps = {
+      nix-build-all.program = pkgs.writeShellApplication {
+        name = "nix-build-all";
+        runtimeInputs = [
+          (pkgs.callPackage inputs.devour-flake {})
+        ];
+        text = ''
+          # Make sure that flake.lock is sync
+          nix flake lock --no-update-lock-file
 
-            # Do a full nix build (all outputs)
-            devour-flake . "$@"
-          '';
-        };
-      }
-      // config.cardanoNix.checks;
+          # Do a full nix build (all outputs)
+          devour-flake . "$@"
+        '';
+      };
+    };
 
     devshells.default.commands = [
       {
diff --git a/checks/testing.nix b/checks/testing.nix
index 130b2e7..05877e1 100644
--- a/checks/testing.nix
+++ b/checks/testing.nix
@@ -1,61 +1,52 @@
 {
-  self,
+  config,
+  lib,
   inputs,
   ...
-}:
-with self.lib; {
-  perSystem = {
-    lib,
-    config,
-    system,
-    ...
-  }: let
-    # create a custom nixpkgs with our flake packages available
-    pkgs = import inputs.nixpkgs {
-      inherit system;
-      overlays = [
-      ];
-    };
-  in {
-    ########################################
-    ## Interface
-    ########################################
-    options.cardanoNix = with lib; {
-      checks = mkOption {
-        type = types.attrsOf types.package;
-        default = {};
-        internal = true;
-      };
-      tests = mkOption {
-        type = types.lazyAttrsOf (types.submodule {
-          options = {
-            systems = mkOption {
-              type = types.listOf types.str;
-            };
-            module = mkOption {
-              type = types.defferedModule;
-            };
+}: let
+  inherit (lib) mkOption types mapAttrs' nameValuePair;
+in {
+  options.cardanoNix = {
+    tests = mkOption {
+      type = types.lazyAttrsOf (types.submodule ({config, ...}: {
+        options = {
+          name = mkOption {
+            default = config._module.args.name;
+            internal = true;
+          };
+          systems = mkOption {
+            type = types.listOf types.str;
           };
-        });
-      };
+          module = mkOption {
+            type = types.deferredModule;
+          };
+        };
+      }));
     };
+  };
 
+  config.perSystem = {
+    lib,
+    system,
+    pkgs,
+    ...
+  }: {
     ########################################
     ## Implementation
     ########################################
-    config.cardanoNix.checks = with lib; let
+    checks = let
       # import the testing framework
-      nixos-lib = import (pkgs.path + "/nixos/lib") {};
+      nixos-lib = import (inputs.nixpkgs.outPath + "/nixos/lib") {};
 
       # examine the `systems` attribute of each test, filtering out any that do not support the current system
-      eachTestForSystem = with lib;
+      tests = with lib;
         filterAttrs
         (_: v: elem system v.systems)
         config.cardanoNix.tests;
     in
       mapAttrs'
       (name: test:
-        nameValuePair "testing-${removeSuffix ".test" name}"
+        nameValuePair "testing-${test.name}"
         (nixos-lib.runTest {
           hostPkgs = pkgs;
 
@@ -63,11 +54,13 @@ with self.lib; {
           defaults.documentation.enable = lib.mkDefault false;
 
           # make self available in test modules and our custom pkgs
-          node.specialArgs = {inherit self pkgs;};
+          node.specialArgs = {
+            inherit pkgs;
+          };
 
           # import all of our flake nixos modules by default
           defaults.imports = [
-            self.nixosModules.default
+            config.flake.nixosModules.default
           ];
 
           # import the test module
@@ -75,76 +68,76 @@ with self.lib; {
         })
         .config
         .result)
-      eachTestForSystem;
+      tests;
 
     ########################################
     ## Commands
     ########################################
-    config.devshells.default.commands = [
+    devshells.default.commands = [
       {
         name = "tests";
-        category = "Testing";
-        help = "Build and run a test";
+        category = "testing";
+        help = "build and run a test";
         command = with lib; ''
-          Help() {
-               # Display Help
-               echo "  Build and run a test"
+          help() {
+               # display help
+               echo "  build and run a test"
                echo
-               echo "  Usage:"
+               echo "  usage:"
                echo "    test <name>"
                echo "    test <name> --interactive"
                echo "    test -s <system> <name>"
                echo
-               echo "  Arguments:"
-               echo "    <name> If a test package is called 'testing-nethermind-basic' then <name> should be 'nethermind-basic'."
+               echo "  arguments:"
+               echo "    <name> if a test package is called 'testing-nethermind-basic' then <name> should be 'nethermind-basic'."
                echo
-               echo "  Options:"
-               echo "    -h --help          Show this screen."
-               echo "    -l --list          Show available tests."
-               echo "    -s --system        Specify the target platform [default: x84_64-linux]."
-               echo "    -i --interactive   Run the test interactively."
+               echo "  options:"
+               echo "    -h --help          show this screen."
+               echo "    -l --list          show available tests."
+               echo "    -s --system        specify the target platform [default: x84_64-linux]."
+               echo "    -i --interactive   run the test interactively."
                echo
           }
 
-          List() {
-            # Display available tests
-            echo "  List of available tests:"
+          list() {
+            # display available tests
+            echo "  list of available tests:"
             echo
-            echo "${strings.concatMapStrings (s: "    - " + s + "\n") (attrsets.mapAttrsToList (name: _: (removePrefix "testing-" name)) config.cardanoNix.checks)}"
+            echo "${strings.concatmapstrings (s: "    - " + s + "\n") (attrsets.mapattrstolist (name: _: (removeprefix "testing-" name)) config.checks)}"
           }
 
-          ARGS=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
-          eval set -- "$ARGS"
+          args=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
+          eval set -- "$args"
 
-          SYSTEM="x86_64-linux"
-          DRIVER_ARGS=()
+          system="x86_64-linux"
+          driver_args=()
 
           while [ $# -gt 0 ]; do
             case "$1" in
-                -i | --interactive) DRIVER_ARGS+=("--interactive"); shift;;
-                -s | --system) SYSTEM="$2"; shift 2;;
-                -h | --help) Help; exit 0;;
-                -l | --list) List; exit 0;;
+                -i | --interactive) driver_args+=("--interactive"); shift;;
+                -s | --system) system="$2"; shift 2;;
+                -h | --help) help; exit 0;;
+                -l | --list) list; exit 0;;
                 -- ) shift; break;;
                 * ) break;;
             esac
           done
 
           if [ $# -eq 0 ]; then
-            # No test name has been provided
-            Help
+            # no test name has been provided
+            help
             exit 1
           fi
 
-          NAME="$1"
+          name="$1"
           shift
 
           # build the test driver
-          DRIVER=$(nix build ".#checks.$SYSTEM.testing-$NAME.driver" --print-out-paths --no-link)
+          driver=$(nix build ".#checks.$system.testing-$name.driver" --print-out-paths --no-link)
 
           # run the test driver, passing any remaining arguments
           set -x
-          ''${DRIVER}/bin/nixos-test-driver "''${DRIVER_ARGS[@]}"
+          ''${driver}/bin/nixos-test-driver "''${driver_args[@]}"
         '';
       }
     ];
diff --git a/flake.nix b/flake.nix
index acd32cc..9b1ef61 100644
--- a/flake.nix
+++ b/flake.nix
@@ -24,8 +24,8 @@
   outputs = inputs @ {flake-parts, ...}:
     flake-parts.lib.mkFlake {
       inherit inputs;
-    }
-    {
+    } {
+      debug = true;
       imports = [
         ./lib
         ./checks
diff --git a/lib/default.nix b/lib/default.nix
index cc1bbec..3c34976 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,8 +1,5 @@
 {lib, ...}: let
   lib' = import ./functions.nix lib;
 in {
-  options.cardanoNix = lib.mkOption {
-    type = lib.types.lazyAttrsOf lib.types.submodule;
-  };
   config.flake.lib = lib';
 }
diff --git a/modules/default.nix b/modules/default.nix
index a70814e..8a7a7ef 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -1,4 +1,4 @@
-{
+{config, ...}: {
   imports = [
   ];
 

From 0978f7e54ca76063c90bbe0a0831435448a79ce9 Mon Sep 17 00:00:00 2001
From: "Alexander V. Nikolaev" <avn@avnik.info>
Date: Fri, 10 Nov 2023 20:52:21 +0200
Subject: [PATCH 5/7] fix: tests script -- fix, cleanup and optimise

---
 checks/testing.nix | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/checks/testing.nix b/checks/testing.nix
index 05877e1..48ca35a 100644
--- a/checks/testing.nix
+++ b/checks/testing.nix
@@ -103,13 +103,13 @@ in {
             # display available tests
             echo "  list of available tests:"
             echo
-            echo "${strings.concatmapstrings (s: "    - " + s + "\n") (attrsets.mapattrstolist (name: _: (removeprefix "testing-" name)) config.checks)}"
+            echo "${concatMapStrings (s: "    - " + s + "\n") (mapAttrsToList (_: test: test.name) config.cardanoNix.tests)}"
           }
 
           args=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
           eval set -- "$args"
 
-          system="x86_64-linux"
+          system="${system}"
           driver_args=()
 
           while [ $# -gt 0 ]; do
@@ -132,12 +132,8 @@ in {
           name="$1"
           shift
 
-          # build the test driver
-          driver=$(nix build ".#checks.$system.testing-$name.driver" --print-out-paths --no-link)
-
-          # run the test driver, passing any remaining arguments
-          set -x
-          ''${driver}/bin/nixos-test-driver "''${driver_args[@]}"
+          # build/run the test driver, passing any remaining arguments
+          nix run ".#checks.$system.testing-$name.driver" -- "''${driver_args[@]}"
         '';
       }
     ];

From 69ed4ece654f27b491f47bf3268b3f80892fa918 Mon Sep 17 00:00:00 2001
From: Andrea Ciceri <andrea.ciceri@autistici.org>
Date: Wed, 15 Nov 2023 13:51:13 +0100
Subject: [PATCH 6/7] refactor: `runTestScript` is an option and it's defined
 in a proper file

---
 checks/run-test.nix |  69 +++++++++++++++++++++
 checks/testing.nix  | 145 ++++++++++++++------------------------------
 2 files changed, 116 insertions(+), 98 deletions(-)
 create mode 100644 checks/run-test.nix

diff --git a/checks/run-test.nix b/checks/run-test.nix
new file mode 100644
index 0000000..0db7ff3
--- /dev/null
+++ b/checks/run-test.nix
@@ -0,0 +1,69 @@
+{
+  writeShellApplication,
+  lib,
+  tests,
+  stdenv,
+  ...
+}:
+writeShellApplication {
+  name = "run-test";
+
+  runtimeInputs = [];
+
+  text = ''
+    cmd_name=$(basename "$0")
+
+    help() {
+      echo "  build and run a test"
+      echo
+      echo "  usage:"
+      echo "    $cmd_name <name>"
+      echo "    $cmd_name <name> --interactive"
+      echo "    $cmd_name -s <system> <name>"
+      echo
+      echo "  arguments:"
+      echo "    <name>"
+      echo
+      echo "  options:"
+      echo "    -h --help          show this screen."
+      echo "    -l --list          show available tests."
+      echo "    -s --system        specify the target platform [default: ${stdenv.system}]."
+      echo "    -i --interactive   run the test interactively."
+      echo
+    }
+
+    list() {
+      echo "  list of available tests:"
+      echo
+      echo "${lib.concatMapStrings (s: "    - " + s + "\n") (lib.mapAttrsToList (_: test: test.name) tests)}"
+    }
+
+    args=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
+    eval set -- "$args"
+
+    system="${stdenv.system}"
+    driver_args=()
+
+    while [ $# -gt 0 ]; do
+    case "$1" in
+      -i | --interactive) driver_args+=("--interactive"); shift;;
+      -s | --system) system="$2"; shift 2;;
+      -h | --help) help; exit 0;;
+      -l | --list) list; exit 0;;
+      -- ) shift; break;;
+      * ) break;;
+      esac
+    done
+
+    if [ $# -eq 0 ]; then
+      help
+      exit 1
+    fi
+
+    name="$1"
+    shift
+
+    # build/run the test driver, passing any remaining arguments
+    nix run ".#checks.$system.testing-$name.driver" -- "''${driver_args[@]}"
+  '';
+}
diff --git a/checks/testing.nix b/checks/testing.nix
index 48ca35a..5bdf8f5 100644
--- a/checks/testing.nix
+++ b/checks/testing.nix
@@ -11,6 +11,7 @@ in {
       type = types.lazyAttrsOf (types.submodule ({config, ...}: {
         options = {
           name = mkOption {
+            type = types.str;
             default = config._module.args.name;
             internal = true;
           };
@@ -30,112 +31,60 @@ in {
     system,
     pkgs,
     ...
-  }: {
-    ########################################
-    ## Implementation
-    ########################################
-    checks = let
-      # import the testing framework
-      nixos-lib = import (inputs.nixpkgs.outPath + "/nixos/lib") {};
+  } @ perSystemArgs: {
+    options.cardanoNix.runTestScript = mkOption {
+      type = types.package;
+      default = pkgs.callPackage ./run-test.nix {inherit (config.cardanoNix) tests;};
+      description = "A convenience script to run tests";
+      internal = true;
+    };
 
-      # examine the `systems` attribute of each test, filtering out any that do not support the current system
-      tests = with lib;
-        filterAttrs
-        (_: v: elem system v.systems)
+    config = let
+      nixos-lib = import (inputs.nixpkgs.outPath + "/nixos/lib") {};
+      tests =
+        lib.filterAttrs
+        (_: v: lib.elem system v.systems)
         config.cardanoNix.tests;
-    in
-      mapAttrs'
-      (name: test:
-        nameValuePair "testing-${test.name}"
-        (nixos-lib.runTest {
-          hostPkgs = pkgs;
-
-          # speed up evaluation by skipping docs
-          defaults.documentation.enable = lib.mkDefault false;
-
-          # make self available in test modules and our custom pkgs
-          node.specialArgs = {
-            inherit pkgs;
-          };
-
-          # import all of our flake nixos modules by default
-          defaults.imports = [
-            config.flake.nixosModules.default
-          ];
-
-          # import the test module
-          imports = [test.module];
-        })
-        .config
-        .result)
-      tests;
 
-    ########################################
-    ## Commands
-    ########################################
-    devshells.default.commands = [
-      {
-        name = "tests";
-        category = "testing";
-        help = "build and run a test";
-        command = with lib; ''
-          help() {
-               # display help
-               echo "  build and run a test"
-               echo
-               echo "  usage:"
-               echo "    test <name>"
-               echo "    test <name> --interactive"
-               echo "    test -s <system> <name>"
-               echo
-               echo "  arguments:"
-               echo "    <name> if a test package is called 'testing-nethermind-basic' then <name> should be 'nethermind-basic'."
-               echo
-               echo "  options:"
-               echo "    -h --help          show this screen."
-               echo "    -l --list          show available tests."
-               echo "    -s --system        specify the target platform [default: x84_64-linux]."
-               echo "    -i --interactive   run the test interactively."
-               echo
-          }
+      inherit (perSystemArgs.config.cardanoNix) runTestScript;
+    in {
+      checks =
+        mapAttrs'
+        (name: test:
+          nameValuePair "testing-${test.name}"
+          (nixos-lib.runTest {
+            hostPkgs = pkgs;
 
-          list() {
-            # display available tests
-            echo "  list of available tests:"
-            echo
-            echo "${concatMapStrings (s: "    - " + s + "\n") (mapAttrsToList (_: test: test.name) config.cardanoNix.tests)}"
-          }
+            # speed up evaluation by skipping docs
+            defaults.documentation.enable = lib.mkDefault false;
 
-          args=$(getopt -o lihs: --long list,interactive,help,system: -n 'tests' -- "$@")
-          eval set -- "$args"
+            # make self available in test modules and our custom pkgs
+            node.specialArgs = {
+              inherit pkgs;
+            };
 
-          system="${system}"
-          driver_args=()
+            # import all of our flake nixos modules by default
+            defaults.imports = [
+              config.flake.nixosModules.default
+            ];
 
-          while [ $# -gt 0 ]; do
-            case "$1" in
-                -i | --interactive) driver_args+=("--interactive"); shift;;
-                -s | --system) system="$2"; shift 2;;
-                -h | --help) help; exit 0;;
-                -l | --list) list; exit 0;;
-                -- ) shift; break;;
-                * ) break;;
-            esac
-          done
+            # import the test module
+            imports = [test.module];
+          })
+          .config
+          .result)
+        tests;
 
-          if [ $# -eq 0 ]; then
-            # no test name has been provided
-            help
-            exit 1
-          fi
+      apps.run-test.program = lib.getExe runTestScript;
 
-          name="$1"
-          shift
-
-          # build/run the test driver, passing any remaining arguments
-          nix run ".#checks.$system.testing-$name.driver" -- "''${driver_args[@]}"
-        '';
-      }
-    ];
+      devshells.default.commands = [
+        {
+          name = "run-test";
+          category = "testing";
+          help = "Run tests";
+          command = "${lib.getExe runTestScript} $@";
+        }
+      ];
+    };
   };
 }

From 4bc6692453eb07421cbdcdbba638794eac68a6bc Mon Sep 17 00:00:00 2001
From: Andrea Ciceri <andrea.ciceri@autistici.org>
Date: Wed, 15 Nov 2023 14:21:34 +0100
Subject: [PATCH 7/7] refactor: `cardanoNix.tests` is a `perSystem` module

---
 checks/testing.nix    | 120 +++++++++++++++++++++++-------------------
 tests/cardano-cli.nix |   3 +-
 tests/default.nix     |   8 +--
 3 files changed, 73 insertions(+), 58 deletions(-)

diff --git a/checks/testing.nix b/checks/testing.nix
index 5bdf8f5..1efe54e 100644
--- a/checks/testing.nix
+++ b/checks/testing.nix
@@ -1,88 +1,100 @@
 {
-  config,
   lib,
   inputs,
+  config,
   ...
 }: let
   inherit (lib) mkOption types mapAttrs' nameValuePair;
+  inherit (config.flake) nixosModules;
 in {
-  options.cardanoNix = {
-    tests = mkOption {
-      type = types.lazyAttrsOf (types.submodule ({config, ...}: {
-        options = {
-          name = mkOption {
-            type = types.str;
-            default = config._module.args.name;
-            internal = true;
-          };
-          systems = mkOption {
-            type = types.listOf types.str;
-          };
-          module = mkOption {
-            type = types.deferredModule;
-          };
-        };
-      }));
-    };
-  };
-
-  config.perSystem = {
+  perSystem = {
+    config,
     lib,
     system,
     pkgs,
     ...
-  } @ perSystemArgs: {
-    options.cardanoNix.runTestScript = mkOption {
-      type = types.package;
-      default = pkgs.callPackage ./run-test.nix {inherit (config.cardanoNix) tests;};
-      description = "A convenience script to run tests";
-      internal = true;
-    };
-
-    config = let
-      nixos-lib = import (inputs.nixpkgs.outPath + "/nixos/lib") {};
-      tests =
-        lib.filterAttrs
-        (_: v: lib.elem system v.systems)
-        config.cardanoNix.tests;
-
-      inherit (perSystemArgs.config.cardanoNix) runTestScript;
-    in {
-      checks =
-        mapAttrs'
-        (name: test:
-          nameValuePair "testing-${test.name}"
-          (nixos-lib.runTest {
+  }: let
+    cfg = config.cardanoNix;
+  in {
+    options.cardanoNix = {
+      tests = mkOption {
+        type = types.lazyAttrsOf (types.submodule ({config, ...}: {
+          options = {
+            name = mkOption {
+              type = types.str;
+              default = config._module.args.name;
+              internal = true;
+            };
+            systems = mkOption {
+              type = types.listOf types.str;
+            };
+            module = mkOption {
+              type = types.deferredModule;
+            };
+            documentation = mkOption {
+              type = types.bool;
+              default = false;
+            };
+            specialArgs = mkOption {
+              type = types.attrsOf types.anything;
+              default = {};
+            };
+          };
+        }));
+      };
+      runTestScript = mkOption {
+        type = types.package;
+        default = pkgs.callPackage ./run-test.nix {inherit (cfg) tests;};
+        description = "A convenience script to run tests";
+      };
+      _nixosLib = mkOption {
+        type = types.anything;
+        default = import (inputs.nixpkgs.outPath + "/nixos/lib") {};
+        internal = true;
+      };
+      _mkCheckFromTest = mkOption {
+        type = types.functionTo types.package;
+        internal = true;
+        default = test:
+          (cfg._nixosLib.runTest {
             hostPkgs = pkgs;
 
-            # speed up evaluation by skipping docs
-            defaults.documentation.enable = lib.mkDefault false;
+            # false by default, it speeds up evaluation by skipping docs generation
+            defaults.documentation.enable = test.documentation;
 
-            # make self available in test modules and our custom pkgs
-            node.specialArgs = {
-              inherit pkgs;
+            node = {
+              inherit (test) specialArgs;
             };
 
             # import all of our flake nixos modules by default
             defaults.imports = [
-              config.flake.nixosModules.default
+              nixosModules.default
             ];
 
             # import the test module
             imports = [test.module];
           })
           .config
-          .result)
-        tests;
+          .result;
+      };
+    };
+
+    config = {
+      checks =
+        mapAttrs'
+        (name: test: nameValuePair "testing-${test.name}" (cfg._mkCheckFromTest test))
+        (lib.filterAttrs
+          (_: v: lib.elem system v.systems)
+          cfg.tests);
 
-      apps.run-test.program = lib.getExe runTestScript;
+      apps.run-test.program = lib.getExe cfg.runTestScript;
 
       devshells.default.commands = [
         {
           name = "run-test";
           category = "testing";
           help = "Run tests";
-          command = "${lib.getExe runTestScript} $@";
+          command = "${lib.getExe cfg.runTestScript} $@";
         }
       ];
     };
diff --git a/tests/cardano-cli.nix b/tests/cardano-cli.nix
index 4e4c6f8..4bc9dc1 100644
--- a/tests/cardano-cli.nix
+++ b/tests/cardano-cli.nix
@@ -1,4 +1,4 @@
-_: {
+{
   cardanoNix.tests = {
     dummy = {
       systems = ["x86_64-linux"];
@@ -16,6 +16,7 @@ _: {
             cardano-ecosystem.cli.enable = true;
           };
         };
+
         testScript = ''
           # FIXME: check for cardano cli, not git
           machine.succeed("git --version")
diff --git a/tests/default.nix b/tests/default.nix
index a943cfe..837ced2 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -1,5 +1,7 @@
 {
-  imports = [
-    ./cardano-cli.nix
-  ];
+  perSystem = _: {
+    imports = [
+      ./cardano-cli.nix
+    ];
+  };
 }