From e77a3898de6b7109cb321ac601809c138b6a0e90 Mon Sep 17 00:00:00 2001
From: Felix Scheinost <31761492+felixscheinost@users.noreply.github.com>
Date: Mon, 19 Jun 2023 18:14:20 +0200
Subject: [PATCH] Replace `remarshal` with `yq` (#26)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remarshal doesn’t seem to be maintained anymore.

It is also affected by upstream bugs like https://github.com/yaml/pyyaml/issues/89 which affects e.g. prometheus-operator CRDs.
---
 lib/helm/chart2json.nix | 52 +++++++++++++++--------------------------
 lib/upstreamables.nix   |  6 ++---
 2 files changed, 22 insertions(+), 36 deletions(-)

diff --git a/lib/helm/chart2json.nix b/lib/helm/chart2json.nix
index b279ac8..02de3e9 100644
--- a/lib/helm/chart2json.nix
+++ b/lib/helm/chart2json.nix
@@ -1,10 +1,8 @@
 {
-  stdenvNoCC,
+  runCommand,
   lib,
   kubernetes-helm,
-  gawk,
-  remarshal,
-  jq,
+  yq,
 }:
 with lib;
   {
@@ -24,37 +22,25 @@ with lib;
     noHooks ? false,
   }: let
     valuesJsonFile = builtins.toFile "${name}-values.json" (builtins.toJSON values);
-  in
-    stdenvNoCC.mkDerivation {
-      name = "${name}.json";
-      buildCommand = ''
-        # template helm file and write resources to yaml
-        helm template "${name}" \
+    # The `helm template` and YAML -> JSON steps are separate `runCommand` derivations for easier debuggability
+    resourcesYaml = runCommand "${name}.yaml" {nativeBuildInputs = [kubernetes-helm];} ''
+      helm template "${name}" \
           ${optionalString (kubeVersion != null) "--kube-version ${kubeVersion}"} \
           ${optionalString (namespace != null) "--namespace ${namespace}"} \
           ${optionalString (values != {}) "-f ${valuesJsonFile}"} \
           ${optionalString includeCRDs "--include-crds"} \
           ${optionalString noHooks "--no-hooks"} \
-          ${chart} >resources.yaml
-
-        # split multy yaml file into multiple files
-        awk 'BEGIN{i=1}{line[i++]=$0}END{j=1;n=0; while (j<i) {if (line[j] ~ /^---/) n++; else print line[j] >>"resource-"n".yaml"; j++}}' resources.yaml
-
-        # join multiple yaml files in jsonl file
-        for file in ./resource-*.yaml
-        do
-          remarshal -i $file -if yaml -of json >>resources.jsonl
-        done
-
-        # convert jsonl file to json array, remove null values and write to $out
-        cat resources.jsonl | jq -Scs 'walk(
-          if type == "object" then
-            with_entries(select(.value != null))
-          elif type == "array" then
-            map(select(. != null))
-          else
-            .
-          end)' > $out
-      '';
-      nativeBuildInputs = [kubernetes-helm gawk remarshal jq];
-    }
+          ${chart} >$out
+    '';
+  in
+    runCommand "${name}.json" {} ''
+      # Remove null values
+      ${yq}/bin/yq -Scs 'walk(
+        if type == "object" then
+          with_entries(select(.value != null))
+        elif type == "array" then
+          map(select(. != null))
+        else
+          .
+        end)' ${resourcesYaml} >$out
+    ''
diff --git a/lib/upstreamables.nix b/lib/upstreamables.nix
index d872af4..f31c690 100644
--- a/lib/upstreamables.nix
+++ b/lib/upstreamables.nix
@@ -6,19 +6,19 @@ with lib; let
   self = {
     importYAML = path:
       importJSON (pkgs.runCommand "yaml-to-json" {} ''
-        ${pkgs.remarshal}/bin/remarshal -i ${path} -if yaml -of json > $out
+        ${pkgs.yq}/bin/yq -c . ${path} > $out
       '');
 
     toYAML = config:
       builtins.readFile (pkgs.runCommand "to-yaml" {} ''
-        ${pkgs.remarshal}/bin/remarshal -i ${pkgs.writeText "to-json" (builtins.toJSON config)} -if json -of yaml > $out
+        ${pkgs.yq}/bin/yq -y . ${pkgs.writeText "to-json" (builtins.toJSON config)}  > $out
       '');
 
     toMultiDocumentYaml = name: documents:
       pkgs.runCommand name {}
       (concatMapStringsSep "\necho --- >> $out\n"
         (
-          d: "${pkgs.remarshal}/bin/remarshal -i ${builtins.toFile "doc" (builtins.toJSON d)} -if json -of yaml >> $out"
+          d: "${pkgs.yq}/bin/yq -y . ${pkgs.writeText "to-json" (builtins.toJSON config)} >> $out"
         )
         documents);