feat: initial v3 rewrite

lib/impl/default.nix

@@ -0,0 +1,12 @@
+{
+  pkgs,
+  lib,
+  cilib,
+}: rec {
+  helpers = import ./helpers.nix {inherit lib pkgs;};
+  mkJobDeps = import ./jobDeps.nix {inherit lib helpers;};
+  mkJobRun = import ./jobRun.nix {inherit lib pkgs helpers;};
+  mkJobPatched = import ./jobPatched.nix {inherit lib helpers;};
+  mkPipeline = import ./pipeline.nix {inherit lib helpers mkJobDeps mkJobRun mkJobPatched;};
+  modules = import ./modules {inherit lib cilib;};
+}

lib/impl/helpers.nix

@@ -0,0 +1,90 @@
+{
+  pkgs,
+  lib,
+} @ args: let
+  inherit (lib) types isAttrs filterAttrs mapAttrs mkOption mkOptionType isType;
+in rec {
+  prepend = key: arr: job: {
+    ${key} = arr ++ (job.${key} or []);
+  };
+  append = key: arr: job: {
+    ${key} = (job.${key} or []) ++ arr;
+  };
+  prependToBeforeScript = prepend "before_script";
+  appendToAfterScript = append "after_script";
+
+  # json is also valid yaml and this removes dependency on jq and/or remarshal
+  # (used in pkgs.formats.json and pkgs.formats.yaml respectively)
+  toYaml = name: value: builtins.toFile name (builtins.toJSON value);
+  toYamlPretty = (pkgs.formats.yaml {}).generate;
+
+  customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
+
+  filterAttrsRec = pred: v:
+    if isAttrs v
+    then filterAttrs pred (mapAttrs (_path: filterAttrsRec pred) v)
+    else v;
+
+  # filter job's variables to either only those containing store paths
+  # or those that do not
+  filterJobVariables = nix: job:
+    lib.concatMapAttrs (
+      name: value:
+        lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) {
+          ${name} = value;
+        }
+    )
+    (job.variables or {});
+
+  deepMerge = lhs: rhs:
+    lhs
+    // rhs
+    // (builtins.mapAttrs (
+        rName: rValue: let
+          lValue = lhs.${rName} or null;
+        in
+          if builtins.isAttrs lValue && builtins.isAttrs rValue
+          then deepMerge lValue rValue
+          else if builtins.isList lValue && builtins.isList rValue
+          then lValue ++ rValue
+          else rValue
+      )
+      rhs);
+
+  unsetType = mkOptionType {
+    name = "unset";
+    description = "unset";
+    descriptionClass = "noun";
+    check = _value: true;
+  };
+  unset = {
+    _type = "unset";
+  };
+  isUnset = isType "unset";
+  unsetOr = types.either unsetType;
+  mkUnsetOption = opts:
+    mkOption (opts
+      // {
+        type = unsetOr opts.type;
+        default = opts.default or unset;
+      });
+
+  filterUnset = value:
+    if builtins.isAttrs value && !builtins.hasAttr "_type" value
+    then let
+      filteredAttrs = builtins.mapAttrs (_n: filterUnset) value;
+    in
+      filterAttrs (_name: value: (!isUnset value)) filteredAttrs
+    else if builtins.isList value
+    then builtins.filter (elem: !isUnset elem) (map filterUnset value)
+    else value;
+
+  # args.pkgs so "pkgs" does not need to be passed all the time
+  stdenvMinimal = args.pkgs.stdenvNoCC.override {
+    cc = null;
+    preHook = "";
+    allowedRequisites = null;
+    initialPath = with args.pkgs; [coreutils findutils];
+    extraNativeBuildInputs = [];
+  };
+}

lib/impl/jobDeps.nix

@@ -0,0 +1,35 @@
+{
+  lib,
+  helpers,
+}: let
+  inherit (lib) concatLines mapAttrsToList makeBinPath;
+  inherit (helpers) filterJobVariables stdenvMinimal;
+in
+  {
+    key,
+    job,
+    nixConfig,
+  }: let
+    variablesWithStorePaths = filterJobVariables true job;
+    variableExports = concatLines (
+      mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
+    );
+    script = ''
+      export PATH="${makeBinPath (nixConfig.deps or [])}:$PATH";
+      # variables containing nix derivations:
+      ${variableExports}
+    '';
+  in
+    stdenvMinimal.mkDerivation {
+      name = "gitlab-ci-job-deps-${key}";
+      dontUnpack = true;
+      installPhase =
+        # sh
+        ''
+          echo '${script}' > $out
+          chmod +x $out
+        '';
+      passthru = {
+        inherit script;
+      };
+    }

lib/impl/jobPatched.nix

@@ -0,0 +1,43 @@
+{
+  lib,
+  helpers,
+}: let
+  inherit (lib) toList optionalAttrs optional;
+  inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables;
+in
+  {
+    key,
+    job,
+    pipelineName,
+    nixConfig,
+  }:
+    job
+    // (optionalAttrs nixConfig.enable (
+      (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job)
+      // (appendToAfterScript ["finalize_nix_ci"] job)
+    ))
+    // optionalAttrs nixConfig.enable (
+      (let
+        variables =
+          (filterJobVariables false job)
+          // optionalAttrs nixConfig.enableRunnerCache {
+            NIX_CI_CACHE_STRATEGY = "runner";
+          };
+      in
+        # filter empty variables
+        optionalAttrs (variables != {}) {
+          inherit variables;
+        })
+      // (let
+        cache =
+          (toList (job.cache or []))
+          ++ (optional nixConfig.enableRunnerCache {
+            key = nixConfig.runnerCacheKey;
+            paths = [".nix-cache/"];
+          });
+      in
+        # filter empty cache
+        optionalAttrs (cache != []) {
+          inherit cache;
+        })
+    )

lib/impl/jobRun.nix

@@ -0,0 +1,47 @@
+{
+  lib,
+  pkgs,
+  helpers,
+}: let
+  inherit (lib) concatLines mapAttrsToList getExe;
+  inherit (helpers) filterJobVariables;
+in
+  {
+    key,
+    job,
+    jobDeps,
+  }: let
+    variablesWithoutStorePaths = filterJobVariables false job;
+    variableExports = concatLines (
+      mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
+    );
+    sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh);
+    actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" ''
+      # set up deps and environment variables containing store paths
+      . ${jobDeps}
+      # normal environment variables
+      ${variableExports}
+      # run before_script, script and after_script
+      echo -e "\e[32mRunning before_script...\e[0m"
+      set -x
+      ${concatLines (job.before_script or [])}
+      { set +x; } 2>/dev/null
+      echo -e "\e[32mRunning script...\e[0m"
+      set -x
+      ${concatLines job.script}
+      { set +x; } 2>/dev/null
+      echo -e "\e[32mRunning after_script...\e[0m"
+      set -x
+      ${concatLines (job.after_script or [])}
+      { set +x; } 2>/dev/null
+    '';
+  in
+    # this way the sandbox helper just needs to be built once
+    pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
+      exec ${getExe sandboxHelper} ${actualJobScript} $@
+    ''
+    // {
+      passthru = {
+        inherit jobDeps actualJobScript;
+      };
+    }

lib/impl/modules/default.nix

@@ -0,0 +1,32 @@
+{
+  lib,
+  cilib,
+}: rec {
+  inherit
+    (import ./root.nix {
+      inherit lib pipelineSubmodule soonixSubmodule;
+    })
+    configSubmodule
+    nixCiSubmodule
+    ;
+  inherit
+    (import ./pipeline.nix {
+      inherit lib cilib jobSubmodule;
+    })
+    pipelineConfigSubmodule
+    pipelineSubmodule
+    ;
+  inherit
+    (import ./job.nix {
+      inherit lib cilib;
+    })
+    jobConfigSubmodule
+    jobSubmodule
+    ;
+  inherit
+    (import ./soonix.nix {
+      inherit lib cilib;
+    })
+    soonixSubmodule
+    ;
+}

lib/impl/modules/job.nix

@@ -0,0 +1,131 @@
+{
+  lib,
+  cilib,
+  ...
+}: let
+  inherit (lib) mkOption types filterAttrs;
+  inherit (cilib.helpers) filterUnset mkUnsetOption;
+in rec {
+  jobConfigSubmodule = {pipelineConfig, ...}: {
+    options = {
+      enable = mkOption {
+        description = "Transform this job to a nix-configured one";
+        type = types.bool;
+        default = pipelineConfig.nixJobsByDefault;
+      };
+      deps = mkOption {
+        description = "Dependencies to inject into the job before running it";
+        type = types.listOf types.package;
+        default = [];
+      };
+      enableRunnerCache = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Cache this job using the GitLab Runner cache.
+          Warning: useful for tiny jobs, but most of the time it just takes an eternity.
+        '';
+      };
+      runnerCacheKey = mkOption {
+        type = types.str;
+        default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
+        description = "Cache key to use for the runner nix cache. Requires enableRunnerCache = true";
+      };
+    };
+  };
+
+  jobSubmodule = {
+    name,
+    config,
+    pipelineName,
+    pipelineConfig,
+    ...
+  }: let
+    #
+    # GITLAB OPTIONS
+    #
+    gitlabOptions = {
+      stage = mkOption {
+        type = types.str;
+      };
+      image = mkUnsetOption {
+        type = types.str;
+      };
+      variables = mkUnsetOption {
+        type = types.attrsOf types.str;
+      };
+      before_script = mkUnsetOption {
+        type = types.listOf types.str;
+      };
+      script = mkOption {
+        type = types.listOf types.str;
+      };
+      after_script = mkUnsetOption {
+        type = types.listOf types.str;
+      };
+      artifacts = mkUnsetOption {
+        type = types.attrs; # TODO: more granular
+        description = '''';
+      };
+      rules = mkUnsetOption {
+        type = types.listOf types.attrs;
+      };
+      allow_failure = mkUnsetOption {
+        type = types.bool;
+      };
+    };
+  in {
+    options =
+      {
+        nix = mkOption {
+          description = "Nix-GitLab-CI config options for this job";
+          type = types.submoduleWith {
+            modules = [jobConfigSubmodule];
+            specialArgs.pipelineConfig = pipelineConfig;
+          };
+          default = {};
+        };
+        finalConfig = mkOption {
+          internal = true;
+          type = types.attrs;
+        };
+        depsDrv = mkOption {
+          internal = true;
+          type = types.package;
+        };
+        runnerDrv = mkOption {
+          internal = true;
+          type = types.package;
+        };
+        packages = mkOption {
+          internal = true;
+          type = types.attrsOf types.package;
+        };
+      }
+      // gitlabOptions;
+    config = let
+      attrsToKeep = builtins.attrNames gitlabOptions;
+    in {
+      finalConfig = cilib.mkJobPatched {
+        key = name;
+        job = filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config);
+        nixConfig = config.nix;
+        inherit pipelineName;
+      };
+      depsDrv = cilib.mkJobDeps {
+        key = name;
+        job = config.finalConfig;
+        nixConfig = config.nix;
+      };
+      runnerDrv = cilib.mkJobRun {
+        key = name;
+        job = config.finalConfig;
+        jobDeps = config.depsDrv;
+      };
+      packages = {
+        "gitlab-ci:pipeline:${pipelineName}:job-deps:${name}" = config.depsDrv;
+        "gitlab-ci:pipeline:${pipelineName}:job:${name}" = config.runnerDrv;
+      };
+    };
+  };
+}

lib/impl/modules/pipeline.nix

@@ -0,0 +1,101 @@
+{
+  lib,
+  cilib,
+  jobSubmodule,
+  ...
+}: let
+  inherit (lib) mkOption types filterAttrs mergeAttrsList pipe mapAttrs;
+  inherit (cilib.helpers) filterUnset mkUnsetOption toYaml toYamlPretty;
+
+  pipelineConfigSubmodule = {rootConfig, ...}: {
+    options = {
+      nixJobsByDefault = mkOption {
+        description = ''
+          Whether to transform all jobs to nix-configured jobs by default.
+          If false, you need to set `nix.enable` for each job you want to be transformed.
+        '';
+        type = types.bool;
+        default = rootConfig.nixJobsByDefault;
+      };
+    };
+  };
+
+  pipelineSubmodule = {
+    name,
+    config,
+    rootConfig,
+    ...
+  }: let
+    #
+    # GITLAB OPTIONS
+    #
+    gitlabOptions = {
+      stages = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        # .pre and .post always exist
+        apply = val: [".pre"] ++ val ++ [".post"];
+      };
+      variables = mkUnsetOption {
+        type = types.attrsOf types.str;
+        description = '''';
+      };
+    };
+  in {
+    _file = ./pipeline.nix;
+    options =
+      {
+        nix = mkOption {
+          description = "Nix-CI config options for this pipeline";
+          type = types.submoduleWith {
+            modules = [pipelineConfigSubmodule];
+            specialArgs.rootConfig = rootConfig;
+          };
+          default = {};
+        };
+        finalConfig = mkOption {
+          description = "Final config of the pipeline";
+          internal = true;
+          type = types.attrs;
+        };
+        packages = mkOption {
+          description = "Final packages for use in CI";
+          internal = true;
+          type = types.attrsOf types.package;
+        };
+        # jobs are nested to make distinguishing them from other keys in the ci config easier
+        jobs = mkOption {
+          description = "Jobs for this pipeline";
+          type = types.attrsOf (types.submoduleWith {
+            modules = [jobSubmodule];
+            specialArgs = {
+              pipelineName = name;
+              pipelineConfig = config.nix;
+            };
+          });
+          default = {};
+        };
+      }
+      // gitlabOptions;
+    config = let
+      attrsToKeep = builtins.attrNames gitlabOptions;
+    in {
+      finalConfig =
+        (filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config))
+        // mapAttrs (_name: value: value.finalConfig) config.jobs;
+      packages =
+        {
+          "gitlab-ci:pipeline:${name}" = pipe config.finalConfig [
+            builtins.toJSON
+            builtins.unsafeDiscardOutputDependency
+            builtins.unsafeDiscardStringContext
+            (toYaml "gitlab-ci-config.json")
+          ];
+          "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-config.yml" config.finalConfig;
+        }
+        // mergeAttrsList (map (job: job.packages) (builtins.attrValues config.jobs));
+    };
+  };
+in {
+  inherit pipelineSubmodule pipelineConfigSubmodule;
+}

lib/impl/modules/root.nix

@@ -0,0 +1,61 @@
+{
+  lib,
+  soonixSubmodule,
+  pipelineSubmodule,
+  ...
+}: let
+  inherit (lib) mkOption types;
+in rec {
+  configSubmodule = {
+    options = {
+      soonix = mkOption {
+        description = "Configure the soonix '.gitlab-ci.yml' generation";
+        type = types.submodule soonixSubmodule;
+        default = {};
+      };
+      nixJobsByDefault = mkOption {
+        description = ''
+          Whether to transform all jobs to nix-configured jobs by default.
+          If false, you need to set `nix.enable` for each job you want to be transformed.
+        '';
+        type = types.bool;
+        default = true;
+      };
+    };
+  };
+
+  nixCiSubmodule = {config, ...}: {
+    options = {
+      config = mkOption {
+        description = "Configuration of Nix-GitLab-CI itself";
+        type = types.submodule configSubmodule;
+        default = {};
+      };
+      pipelines = mkOption {
+        description = "Defines all pipelines";
+        type = types.attrsOf (types.submoduleWith {
+          modules = [pipelineSubmodule];
+          specialArgs.rootConfig = config.config;
+        });
+        default = {};
+      };
+
+      packages = mkOption {
+        description = "Final packages for use in CI";
+        internal = true;
+        type = types.attrsOf types.package;
+      };
+      soonix = mkOption {
+        description = "Soonix config for .gitlab-ci.yml";
+        internal = true;
+        type = types.attrs;
+      };
+    };
+    config = {
+      packages = lib.fold (pipeline: acc: acc // pipeline) {} (
+        map (pipeline: pipeline.packages) (builtins.attrValues config.pipelines)
+      );
+      soonix = config.config.soonix.finalConfig;
+    };
+  };
+}

lib/impl/modules/soonix.nix

@@ -0,0 +1,61 @@
+{
+  lib,
+  cilib,
+  ...
+}: let
+  inherit (lib) mkOption types;
+in {
+  soonixSubmodule = {config, ...}: {
+    options = {
+      componentVersion = mkOption {
+        description = "CI/CD component version. Also get's passed to inputs → version";
+        type = types.str;
+        default = cilib.version;
+      };
+      componentUrl = mkOption {
+        description = "CI/CD component url";
+        type = types.str;
+        default = "gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci";
+      };
+      componentInputs = mkOption {
+        description = "Extra inputs to pass to the CI/CD component";
+        type = types.attrs;
+        default = {};
+      };
+      extraData = mkOption {
+        description = "Extra data to include in the .gitlab-ci.yml file";
+        type = types.attrs;
+        default = {};
+      };
+
+      finalConfig = mkOption {
+        internal = true;
+        type = types.attrs;
+      };
+    };
+    config.finalConfig = {
+      opts.format = "yaml";
+      hook = {
+        mode = "copy";
+        gitignore = false;
+      };
+      output = ".gitlab-ci.yml";
+      generator = "nix";
+      data =
+        cilib.helpers.deepMerge
+        {
+          include = [
+            {
+              component = "${config.componentUrl}@${config.componentVersion}";
+              inputs =
+                {
+                  version = config.componentVersion;
+                }
+                // config.componentInputs;
+            }
+          ];
+        }
+        config.extraData;
+    };
+  };
+}

lib/impl/pipeline.nix

@@ -0,0 +1,61 @@
+{
+  lib,
+  helpers,
+  mkJobDeps,
+  mkJobRun,
+  mkJobPatched,
+}: let
+  inherit (lib) assertMsg;
+  inherit (helpers) filterAttrsRec customMapAttrs toYaml toYamlPretty;
+in
+  {
+    name,
+    pipeline,
+    nixConfig,
+  }: let
+    jobs = filterAttrsRec (_n: v: v != null) pipeline.jobs;
+    rest = filterAttrsRec (_n: v: v != null) (builtins.removeAttrs pipeline ["jobs"]);
+    # this allows us to nix build this to get all the mentioned dependencies from the binary cache
+    # pro: we don't have to download everything, just the deps for the current job
+    # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
+    # we can support different architectures between runners (eg. the arch of the initial runner does not matter)
+    jobsMappedForDeps =
+      customMapAttrs (key: job: {
+        name = "gitlab-ci:pipeline:${name}:job-deps:${key}";
+        value = mkJobDeps {inherit key job nixConfig;};
+      })
+      jobs;
+    # allows the user to directly run the script
+    jobsMappedForScript =
+      customMapAttrs (key: job: {
+        name = "gitlab-ci:pipeline:${name}:job:${key}";
+        value = mkJobRun {
+          inherit key job nixConfig;
+          jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}";
+        };
+      })
+      jobs;
+    # build the deps specific for this job before anything, this way the deps should be fetched from the cache
+    jobsPatched =
+      customMapAttrs (key: job: {
+        name = key;
+        value = assert assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
+          mkJobPatched {
+            inherit key job nixConfig;
+            pipelineName = name;
+          };
+      })
+      jobs;
+  in {
+    packages =
+      #  gitlab-ci:pipeline:<name>
+      #  gitlab-ci:pipeline:<name>:job:<name>
+      #  gitlab-ci:pipeline:<name>:job-deps:<name>
+      {
+        "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched);
+        "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-${name}.yml" (rest // jobsPatched);
+      }
+      // jobsMappedForDeps
+      // jobsMappedForScript;
+    finalConfig = rest // jobsPatched;
+  }

lib/impl/sandbox_helper.sh

@@ -0,0 +1,73 @@
+echo -e "\e[32mSetting up...\e[0m"
+
+actualJobScript=$1
+shift
+
+INCLUDE_DIRTY=false
+NO_SANDBOX=false
+KEEP_TMP=false
+KEEP_ENV=""
+# parse flags
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --include-dirty)
+      INCLUDE_DIRTY=true
+      shift
+      ;;
+    --no-sandbox)
+      NO_SANDBOX=true
+      shift
+      ;;
+    --keep-tmp)
+      KEEP_TMP=true
+      shift
+      ;;
+    --keep-env)
+      KEEP_ENV="$2"
+      shift 2
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [ "$NO_SANDBOX" = false ]; then
+  echo "Running with simple sandboxing"
+  if [ "$KEEP_TMP" = false ]; then
+    trap "rm -rf '$TMPDIR'" EXIT
+  else
+    echo "Temp dir will be preserved at: $TMPDIR"
+  fi
+
+  # check if dirty
+  DIRTY_PATCH=""
+  if ! git diff --quiet && ! git diff --staged --quiet; then
+    echo "Warning: working tree is dirty."
+    DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch")
+    git diff --staged > "$DIRTY_PATCH"
+    trap "rm -f '$DIRTY_PATCH'" EXIT
+  fi
+  TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX")
+  git clone . $TMPDIR
+  pushd $TMPDIR >/dev/null
+  if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then
+    echo "Copying dirty changes..."
+    git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes"
+  fi
+
+  echo "Running job in $TMPDIR"
+  env -i $(
+    if [[ -n "$KEEP_ENV" ]]; then
+      IFS=',' read -ra VARS <<< "$KEEP_ENV"
+      for var in "${VARS[@]}"; do
+        printf '%s=%q ' "$var" "${!var}"
+      done
+    fi
+  ) bash $actualJobScript
+  popd >/dev/null
+else
+  exec $actualJobScript
+fi
+