feat: improve testing

.travis.yml

@@ -1,2 +1,6 @@
 language: nix
-script: nix build -f ./tests v1_13.testing.result --arg e2e false
+script:
+- nix-env -iA nixpkgs.jq
+- nix build -f ./release.nix test-results --arg e2e false
+- cat result | jq '.'
+- nix eval -f ./release.nix --arg e2e false tests-check

modules/testing.nix

@@ -245,14 +245,14 @@ in {
 
     testing.result = mkOption {
       description = "Testing result";
-      type = types.package;
-      default = pkgs.writeText "testing-report.json" (builtins.toJSON {
+      type = types.attrs;
+      default = {
         success = cfg.success;
         tests = map (test: {
           inherit (test) name description evaled success test;
           assertions = moduleToAttrs test.assertions;
         }) (filter (test: test.enable) cfg.tests);
-      });
+      };
     };
   };
 }

release.nix

@@ -1,4 +1,6 @@
-{ pkgs ? import <nixpkgs> {} }:
+{ pkgs ? import <nixpkgs> {}, lib ? pkgs.lib, e2e ? true, throwError ? true }:
+
+with lib;
 
 let
   kubenix = import ./. { inherit pkgs; };
@@ -15,7 +17,11 @@ let
     inherit pkgs;
     inherit (pkgs) lib;
   };
-in {
+
+  runK8STests = k8sVersion: pkgs.recurseIntoAttrs (import ./tests {
+    inherit pkgs lib kubenix k8sVersion e2e throwError;
+  });
+in rec {
   generate.k8s = pkgs.linkFarm "k8s-generated.nix" [{
     name = "v1.7.nix";
     path = generateK8S "${pkgs.fetchFromGitHub {
@@ -79,9 +85,25 @@ in {
     path = generateIstio;
   }];
 
-  tests = import ./tests {
-    inherit pkgs lib kubenix;
+  tests = {
+    k8s-1_7 = runK8STests "1.7";
+    k8s-1_8 = runK8STests "1.8";
+    k8s-1_9 = runK8STests "1.9";
+    k8s-1_10 = runK8STests "1.10";
+    k8s-1_11 = runK8STests "1.11";
+    k8s-1_12 = runK8STests "1.12";
+    k8s-1_13 = runK8STests "1.13";
   };
 
+  test-results = pkgs.writeText "kubenix-test-result.json" (builtins.toJSON {
+    success = all (test: test.success) (attrValues tests);
+    results = mapAttrs (_: test: test.result) tests;
+  });
+
+  tests-check =
+    if !(all (test: test.success) (attrValues tests))
+    then throw "tests failed"
+    else true;
+
   examples = import ./examples {};
 }

tests/default.nix

@@ -1,7 +1,7 @@
 { pkgs ? import <nixpkgs> {}
 , lib ? pkgs.lib
 , kubenix ? import ../. { inherit pkgs lib; }
-, k8sVersions ? ["1.7" "1.8" "1.9" "1.10" "1.11" "1.12" "1.13"]
+, k8sVersion ? "1.13"
 
 # whether any testing error should throw an error
 , throwError ? true
@@ -12,9 +12,7 @@ with lib;
 let
   images = pkgs.callPackage ./images.nix {};
 
-  tests = listToAttrs (map (version: let
-    version' = replaceStrings ["."] ["_"] version;
-  in nameValuePair "v${version'}" (kubenix.evalModules {
+  test = (kubenix.evalModules {
     modules = [
       kubenix.modules.testing
 
@@ -37,7 +35,7 @@ let
         ];
         testing.defaults = ({kubenix, ...}: {
           imports = [kubenix.modules.k8s];
-          kubernetes.version = version;
+          kubernetes.version = k8sVersion;
           _module.args.images = images;
         });
       }
@@ -48,5 +46,5 @@ let
     specialArgs = {
       inherit kubenix;
     };
-  }).config) k8sVersions);
-in tests
+  }).config;
+in test.testing

tests/helm/simple.nix

@@ -29,11 +29,12 @@ let
     finalImageTag = "latest";
   };
 in {
-  imports = [ kubenix.modules.test kubenix.modules.helm ];
+  imports = [ kubenix.modules.test kubenix.modules.helm kubenix.modules.k8s ];
 
   test = {
     name = "helm-simple";
     description = "Simple k8s testing wheter name, apiVersion and kind are preset";
+    enable = builtins.compareVersions config.kubernetes.version "1.8" >= 0;
     assertions = [{
       message = "should have generated resources";
       assertion =

tests/k8s/deployment.nix

@@ -12,8 +12,16 @@ in {
     name = "k8s-deployment";
     description = "Simple k8s testing a simple deployment";
     assertions = [{
-      message = "should have correct apiVersion and kind set";
-      assertion = cfg.apiVersion == "apps/v1" && cfg.kind == "Deployment";
+      message = "should have correct apiVersion and kind set for 1.8+";
+      assertion =
+        if ((builtins.compareVersions config.kubernetes.version "1.7") <= 0)
+        then cfg.apiVersion == "apps/v1beta1"
+        else if ((builtins.compareVersions config.kubernetes.version "1.8") <= 0)
+        then cfg.apiVersion == "apps/v1beta2"
+        else cfg.apiVersion == "apps/v1";
+    } {
+      message = "should have corrent kind set";
+      assertion = cfg.kind == "Deployment";
     } {
       message = "should have replicas set";
       assertion = cfg.spec.replicas == 10;