diff --git a/.travis.yml b/.travis.yml index 4c6be18..977d9e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ language: nix script: - nix-env -iA nixpkgs.jq -- nix build -f ./release.nix test-results --arg e2e false -- cat result | jq '.' - nix eval -f ./release.nix --arg e2e false test-check diff --git a/ci.nix b/ci.nix index 41fa5ea..fc062a2 100644 --- a/ci.nix +++ b/ci.nix @@ -6,6 +6,7 @@ let release = import ./release.nix { inherit pkgs lib; + e2e = false; nixosPath = "${nixpkgsSrc}/nixos"; }; -in with lib; release.test-results +in with lib; release.tests diff --git a/modules/testing.nix b/modules/testing.nix index b2d089c..06418d2 100644 --- a/modules/testing.nix +++ b/modules/testing.nix @@ -5,6 +5,8 @@ with lib; let cfg = config.testing; + toJSONFile = content: builtins.toFile "json" (builtins.toJSON content); + nixosTesting = import "${nixosPath}/lib/testing.nix" { inherit pkgs; system = "x86_64-linux"; @@ -155,7 +157,7 @@ let }; evaled = mkOption { - description = "Wheter test was evaled"; + description = "Test evaulation result"; type = types.nullOr types.attrs; internal = true; }; @@ -180,16 +182,26 @@ let default = null; }; - generated = mkOption { - description = "Generated resources"; - type = types.nullOr types.package; - default = null; + result = mkOption { + description = "Test result"; + type = types.package; }; }; config = mkMerge [{ inherit evaled; inherit (test) name description enable; + result = pkgs.runCommand "${cfg.name}-${config.name}-test.json" { + buildInputs = [ pkgs.jshon pkgs.jq ]; + } '' + jshon -n object \ + -s "${config.name}" -i name \ + -s "${config.description}" -i description \ + -n "${if config.success then "true" else "false"}" -i success \ + ${if config.test == null then "-n null" else "-s '${config.test}'"} -i test > result.json + + jq -s '.[0].assertions = .[1] | .[0]' result.json ${toJSONFile (map (getAttrs ["assertion" "message"]) config.assertions)} > $out + ''; } (mkIf (config.evaled != null) { inherit (evaled.config.test) assertions; success = all (el: el.assertion) config.assertions; @@ -199,12 +211,16 @@ let name = config.name; inherit (evaled.config.test) testScript extraConfiguration; } else null; - generated = mkIf (hasAttr "kubernetes" evaled.config) - (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated)); })]; }; in { options = { + testing.name = mkOption { + description = "Testing suite name"; + type = types.str; + default = "default"; + }; + testing.throwError = mkOption { description = "Whether to throw error"; type = types.bool; @@ -227,7 +243,7 @@ in { }; testing.tests = mkOption { - description = "Attribute set of test cases"; + description = "List of test cases"; default = []; type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule testOptions)); apply = tests: filter (test: test.enable) tests; @@ -251,17 +267,24 @@ in { default = all (test: test.success) cfg.tests; }; + testing.results = mkOption { + description = "Test results"; + type = types.attrsOf types.package; + default = mapAttrs (_: t: t.result) cfg.testsByName; + }; + testing.result = mkOption { - description = "Testing result"; - type = types.attrs; - default = { - success = cfg.success; - tests = map (test: { - inherit (test) name description success test; - assertions = moduleToAttrs test.assertions; - generated = test.generated; - }) (filter (test: test.enable) cfg.tests); - }; + description = "Testing results"; + type = types.package; + default = pkgs.runCommand "${cfg.name}-test-results.json" { + buildInputs = [ pkgs.jq ]; + } '' + jq -s -r '.' ${concatMapStringsSep " " (t: t.result) cfg.tests} > tests.json + jq -n \ + --rawfile tests tests.json \ + --argjson success `jq -s -r 'if all(.success) == true then true else false end' ${concatMapStringsSep " " (t: t.result) cfg.tests}` \ + '{"success": $success, "tests": $tests | fromjson }' > $out + ''; }; }; } diff --git a/release.nix b/release.nix index e71297b..1869ccf 100644 --- a/release.nix +++ b/release.nix @@ -1,4 +1,4 @@ -{ pkgs ? import {}, nixosPath ? , lib ? pkgs.lib +{ pkgs ? import {}, nixosPath ? toString , lib ? pkgs.lib , e2e ? true, throwError ? true }: with lib; @@ -19,9 +19,11 @@ let inherit (pkgs) lib; }; - runK8STests = k8sVersion: pkgs.recurseIntoAttrs (import ./tests { - inherit pkgs lib kubenix k8sVersion e2e throwError nixosPath; - }); + runK8STests = k8sVersion: pkgs.recurseIntoAttrs (getAttrs ["result" "results" "success"] + (import ./tests { + inherit pkgs lib kubenix k8sVersion e2e throwError nixosPath; + }) + ); in rec { generate.k8s = pkgs.linkFarm "k8s-generated.nix" [{ name = "v1.7.nix"; @@ -86,7 +88,7 @@ in rec { path = generateIstio; }]; - tests = { + tests = pkgs.recurseIntoAttrs { k8s-1_7 = runK8STests "1.7"; k8s-1_8 = runK8STests "1.8"; k8s-1_9 = runK8STests "1.9"; @@ -96,13 +98,8 @@ in rec { k8s-1_13 = runK8STests "1.13"; }; - test-results = pkgs.writeText "kubenix-test-result.json" (builtins.toJSON { - success = all (test: test.success) (attrValues tests); - results = mapAttrs (_: test: test.result) tests; - }); - test-check = - if !(all (test: test.success) (attrValues tests)) + if !(all (test: if isAttrs test then test.success else true) (attrValues tests)) then throw "tests failed" else true; diff --git a/tests/default.nix b/tests/default.nix index 5912a47..93f7132 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -2,7 +2,7 @@ , lib ? pkgs.lib , kubenix ? import ../. { inherit pkgs lib; } , k8sVersion ? "1.13" -, nixosPath ? +, nixosPath ? toString # whether any testing error should throw an error , throwError ? true @@ -18,6 +18,7 @@ let kubenix.modules.testing { + testing.name = "k8s-${k8sVersion}"; testing.throwError = throwError; testing.e2e = e2e; testing.tests = [ @@ -47,4 +48,4 @@ let inherit kubenix nixosPath; }; }).config; -in test.testing +in pkgs.recurseIntoAttrs test.testing diff --git a/tests/images.nix b/tests/images.nix index ee6eb9a..80bea14 100644 --- a/tests/images.nix +++ b/tests/images.nix @@ -30,7 +30,7 @@ with lib; tag = "latest"; contents = [pkgs.nginx]; extraCommands = '' - mkdir etc + mkdir -p etc chmod u+w etc echo "nginx:x:1000:1000::/:" > etc/passwd echo "nginx:x:1000:nginx" > etc/group