improve testing

.travis.yml

@@ -1,6 +1,4 @@
 language: nix
 script:
 - nix-env -iA nixpkgs.jq
-- nix build -f ./release.nix test-results --arg e2e false
-- cat result | jq '.'
 - nix eval -f ./release.nix --arg e2e false test-check

ci.nix

@@ -6,6 +6,7 @@ let
 
   release = import ./release.nix {
     inherit pkgs lib;
+    e2e = false;
     nixosPath = "${nixpkgsSrc}/nixos";
   };
-in with lib; release.test-results
+in with lib; release.tests

modules/testing.nix

@@ -5,6 +5,8 @@ with lib;
 let
   cfg = config.testing;
 
+  toJSONFile = content: builtins.toFile "json" (builtins.toJSON content);
+
   nixosTesting = import "${nixosPath}/lib/testing.nix" {
     inherit pkgs;
     system = "x86_64-linux";
@@ -155,7 +157,7 @@ let
       };
 
       evaled = mkOption {
-        description = "Wheter test was evaled";
+        description = "Test evaulation result";
         type = types.nullOr types.attrs;
         internal = true;
       };
@@ -180,16 +182,26 @@ let
         default = null;
       };
 
-      generated = mkOption {
-        description = "Generated resources";
-        type = types.nullOr types.package;
-        default = null;
+      result = mkOption {
+        description = "Test result";
+        type = types.package;
       };
     };
 
     config = mkMerge [{
       inherit evaled;
       inherit (test) name description enable;
+      result = pkgs.runCommand "${cfg.name}-${config.name}-test.json" {
+        buildInputs = [ pkgs.jshon pkgs.jq ];
+      } ''
+        jshon -n object \
+          -s "${config.name}" -i name \
+          -s "${config.description}" -i description \
+          -n "${if config.success then "true" else "false"}" -i success \
+          ${if config.test == null then "-n null" else "-s '${config.test}'"} -i test > result.json
+
+        jq -s '.[0].assertions = .[1] | .[0]' result.json ${toJSONFile (map (getAttrs ["assertion" "message"]) config.assertions)} > $out
+      '';
     } (mkIf (config.evaled != null) {
       inherit (evaled.config.test) assertions;
       success = all (el: el.assertion) config.assertions;
@@ -199,12 +211,16 @@ let
           name = config.name;
           inherit (evaled.config.test) testScript extraConfiguration;
         } else null;
-      generated = mkIf (hasAttr "kubernetes" evaled.config)
-        (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated));
     })];
   };
 in {
   options = {
+    testing.name = mkOption {
+      description = "Testing suite name";
+      type = types.str;
+      default = "default";
+    };
+
     testing.throwError = mkOption {
       description = "Whether to throw error";
       type = types.bool;
@@ -227,7 +243,7 @@ in {
     };
 
     testing.tests = mkOption {
-      description = "Attribute set of test cases";
+      description = "List of test cases";
       default = [];
       type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule testOptions));
       apply = tests: filter (test: test.enable) tests;
@@ -251,17 +267,24 @@ in {
       default = all (test: test.success) cfg.tests;
     };
 
+    testing.results = mkOption {
+      description = "Test results";
+      type = types.attrsOf types.package;
+      default = mapAttrs (_: t: t.result) cfg.testsByName;
+    };
+
     testing.result = mkOption {
-      description = "Testing result";
-      type = types.attrs;
-      default = {
-        success = cfg.success;
-        tests = map (test: {
-          inherit (test) name description success test;
-          assertions = moduleToAttrs test.assertions;
-          generated = test.generated;
-        }) (filter (test: test.enable) cfg.tests);
-      };
+      description = "Testing results";
+      type = types.package;
+      default = pkgs.runCommand "${cfg.name}-test-results.json" {
+        buildInputs = [ pkgs.jq ];
+      } ''
+        jq -s -r '.' ${concatMapStringsSep " " (t: t.result) cfg.tests} > tests.json
+        jq -n \
+          --rawfile tests tests.json \
+          --argjson success `jq -s -r 'if all(.success) == true then true else false end' ${concatMapStringsSep " " (t: t.result) cfg.tests}` \
+          '{"success": $success, "tests": $tests | fromjson }' > $out
+      '';
     };
   };
 }

release.nix

@@ -1,4 +1,4 @@
-{ pkgs ? import <nixpkgs> {}, nixosPath ? <nixpkgs/nixos>, lib ? pkgs.lib
+{ pkgs ? import <nixpkgs> {}, nixosPath ? toString <nixpkgs/nixos>, lib ? pkgs.lib
 , e2e ? true, throwError ? true }:
 
 with lib;
@@ -19,9 +19,11 @@ let
     inherit (pkgs) lib;
   };
 
-  runK8STests = k8sVersion: pkgs.recurseIntoAttrs (import ./tests {
-    inherit pkgs lib kubenix k8sVersion e2e throwError nixosPath;
-  });
+  runK8STests = k8sVersion: pkgs.recurseIntoAttrs (getAttrs ["result" "results" "success"]
+    (import ./tests {
+      inherit pkgs lib kubenix k8sVersion e2e throwError nixosPath;
+    })
+  );
 in rec {
   generate.k8s = pkgs.linkFarm "k8s-generated.nix" [{
     name = "v1.7.nix";
@@ -86,7 +88,7 @@ in rec {
     path = generateIstio;
   }];
 
-  tests = {
+  tests = pkgs.recurseIntoAttrs {
     k8s-1_7 = runK8STests "1.7";
     k8s-1_8 = runK8STests "1.8";
     k8s-1_9 = runK8STests "1.9";
@@ -96,13 +98,8 @@ in rec {
     k8s-1_13 = runK8STests "1.13";
   };
 
-  test-results = pkgs.writeText "kubenix-test-result.json" (builtins.toJSON {
-    success = all (test: test.success) (attrValues tests);
-    results = mapAttrs (_: test: test.result) tests;
-  });
-
   test-check =
-    if !(all (test: test.success) (attrValues tests))
+    if !(all (test: if isAttrs test then test.success else true) (attrValues tests))
     then throw "tests failed"
     else true;
 

tests/default.nix

@@ -2,7 +2,7 @@
 , lib ? pkgs.lib
 , kubenix ? import ../. { inherit pkgs lib; }
 , k8sVersion ? "1.13"
-, nixosPath ? <nixpkgs/nixos>
+, nixosPath ? toString <nixpkgs/nixos>
 
 # whether any testing error should throw an error
 , throwError ? true
@@ -18,6 +18,7 @@ let
       kubenix.modules.testing
 
       {
+        testing.name = "k8s-${k8sVersion}";
         testing.throwError = throwError;
         testing.e2e = e2e;
         testing.tests = [
@@ -47,4 +48,4 @@ let
       inherit kubenix nixosPath;
     };
   }).config;
-in test.testing
+in pkgs.recurseIntoAttrs test.testing

tests/images.nix

@@ -30,7 +30,7 @@ with lib;
     tag = "latest";
     contents = [pkgs.nginx];
     extraCommands = ''
-      mkdir etc
+      mkdir -p etc
       chmod u+w etc
       echo "nginx:x:1000:1000::/:" > etc/passwd
       echo "nginx:x:1000:nginx" > etc/group