From 987c6e41735911db2ed0f45ce475b95ea006dcb0 Mon Sep 17 00:00:00 2001
From: Jaka Hudoklin <jaka@gatehub.net>
Date: Fri, 8 Mar 2019 00:39:09 +0100
Subject: [PATCH] feat: improve testing

---
 .travis.yml              |  6 +++++-
 modules/testing.nix      |  6 +++---
 release.nix              | 30 ++++++++++++++++++++++++++----
 tests/default.nix        | 12 +++++-------
 tests/helm/simple.nix    |  3 ++-
 tests/k8s/deployment.nix | 12 ++++++++++--
 6 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0b4329b..87ddabe 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,2 +1,6 @@
 language: nix
-script: nix build -f ./tests v1_13.testing.result --arg e2e false
+script:
+- nix-env -iA nixpkgs.jq
+- nix build -f ./release.nix test-results --arg e2e false
+- cat result | jq '.'
+- nix eval -f ./release.nix --arg e2e false tests-check
diff --git a/modules/testing.nix b/modules/testing.nix
index 6027149..a4b06c0 100644
--- a/modules/testing.nix
+++ b/modules/testing.nix
@@ -245,14 +245,14 @@ in {
 
     testing.result = mkOption {
       description = "Testing result";
-      type = types.package;
-      default = pkgs.writeText "testing-report.json" (builtins.toJSON {
+      type = types.attrs;
+      default = {
         success = cfg.success;
         tests = map (test: {
           inherit (test) name description evaled success test;
           assertions = moduleToAttrs test.assertions;
         }) (filter (test: test.enable) cfg.tests);
-      });
+      };
     };
   };
 }
diff --git a/release.nix b/release.nix
index 2314d2b..c5cb613 100644
--- a/release.nix
+++ b/release.nix
@@ -1,4 +1,6 @@
-{ pkgs ? import <nixpkgs> {} }:
+{ pkgs ? import <nixpkgs> {}, lib ? pkgs.lib, e2e ? true, throwError ? true }:
+
+with lib;
 
 let
   kubenix = import ./. { inherit pkgs; };
@@ -15,7 +17,11 @@ let
     inherit pkgs;
     inherit (pkgs) lib;
   };
-in {
+
+  runK8STests = k8sVersion: pkgs.recurseIntoAttrs (import ./tests {
+    inherit pkgs lib kubenix k8sVersion e2e throwError;
+  });
+in rec {
   generate.k8s = pkgs.linkFarm "k8s-generated.nix" [{
     name = "v1.7.nix";
     path = generateK8S "${pkgs.fetchFromGitHub {
@@ -79,9 +85,25 @@ in {
     path = generateIstio;
   }];
 
-  tests = import ./tests {
-    inherit pkgs lib kubenix;
+  tests = {
+    k8s-1_7 = runK8STests "1.7";
+    k8s-1_8 = runK8STests "1.8";
+    k8s-1_9 = runK8STests "1.9";
+    k8s-1_10 = runK8STests "1.10";
+    k8s-1_11 = runK8STests "1.11";
+    k8s-1_12 = runK8STests "1.12";
+    k8s-1_13 = runK8STests "1.13";
   };
 
+  test-results = pkgs.writeText "kubenix-test-result.json" (builtins.toJSON {
+    success = all (test: test.success) (attrValues tests);
+    results = mapAttrs (_: test: test.result) tests;
+  });
+
+  tests-check =
+    if !(all (test: test.success) (attrValues tests))
+    then throw "tests failed"
+    else true;
+
   examples = import ./examples {};
 }
diff --git a/tests/default.nix b/tests/default.nix
index 251b69e..2634e15 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -1,7 +1,7 @@
 { pkgs ? import <nixpkgs> {}
 , lib ? pkgs.lib
 , kubenix ? import ../. { inherit pkgs lib; }
-, k8sVersions ? ["1.7" "1.8" "1.9" "1.10" "1.11" "1.12" "1.13"]
+, k8sVersion ? "1.13"
 
 # whether any testing error should throw an error
 , throwError ? true
@@ -12,9 +12,7 @@ with lib;
 let
   images = pkgs.callPackage ./images.nix {};
 
-  tests = listToAttrs (map (version: let
-    version' = replaceStrings ["."] ["_"] version;
-  in nameValuePair "v${version'}" (kubenix.evalModules {
+  test = (kubenix.evalModules {
     modules = [
       kubenix.modules.testing
 
@@ -37,7 +35,7 @@ let
         ];
         testing.defaults = ({kubenix, ...}: {
           imports = [kubenix.modules.k8s];
-          kubernetes.version = version;
+          kubernetes.version = k8sVersion;
           _module.args.images = images;
         });
       }
@@ -48,5 +46,5 @@ let
     specialArgs = {
       inherit kubenix;
     };
-  }).config) k8sVersions);
-in tests
+  }).config;
+in test.testing
diff --git a/tests/helm/simple.nix b/tests/helm/simple.nix
index 98f05d5..85e04c2 100644
--- a/tests/helm/simple.nix
+++ b/tests/helm/simple.nix
@@ -29,11 +29,12 @@ let
     finalImageTag = "latest";
   };
 in {
-  imports = [ kubenix.modules.test kubenix.modules.helm ];
+  imports = [ kubenix.modules.test kubenix.modules.helm kubenix.modules.k8s ];
 
   test = {
     name = "helm-simple";
     description = "Simple k8s testing wheter name, apiVersion and kind are preset";
+    enable = builtins.compareVersions config.kubernetes.version "1.8" >= 0;
     assertions = [{
       message = "should have generated resources";
       assertion =
diff --git a/tests/k8s/deployment.nix b/tests/k8s/deployment.nix
index d2288fd..44ee5f3 100644
--- a/tests/k8s/deployment.nix
+++ b/tests/k8s/deployment.nix
@@ -12,8 +12,16 @@ in {
     name = "k8s-deployment";
     description = "Simple k8s testing a simple deployment";
     assertions = [{
-      message = "should have correct apiVersion and kind set";
-      assertion = cfg.apiVersion == "apps/v1" && cfg.kind == "Deployment";
+      message = "should have correct apiVersion and kind set for 1.8+";
+      assertion =
+        if ((builtins.compareVersions config.kubernetes.version "1.7") <= 0)
+        then cfg.apiVersion == "apps/v1beta1"
+        else if ((builtins.compareVersions config.kubernetes.version "1.8") <= 0)
+        then cfg.apiVersion == "apps/v1beta2"
+        else cfg.apiVersion == "apps/v1";
+    } {
+      message = "should have corrent kind set";
+      assertion = cfg.kind == "Deployment";
     } {
       message = "should have replicas set";
       assertion = cfg.spec.replicas == 10;