feat: add gitlab source for gitlab package registry

cleanup http source

flake.nix

@@ -4,7 +4,7 @@
 
   outputs = { self, nixpkgs, systems }:
   let
-    version = "0.5.1";
+    version = "0.5.2";
     forEachSystem = nixpkgs.lib.genAttrs (import systems);
     nixpkgsFor = forEachSystem (system: import nixpkgs { inherit system; });
   in {

pkg/git.go

@@ -17,8 +17,6 @@ package pkg
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -97,11 +95,9 @@ func remoteResolveRef(ctx context.Context, remote string, ref string) (string, e
 func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (string, error) {
 	destPath := path.Join(dir, name)
 
-	pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1))))
+	tmpDir, err := CreateTempDir(name, dir, version)
-	// using 16 bytes should be a good middle ground between length and collision resistance
-	tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16]))
 	if err != nil {
-		return "", errors.Wrap(err, "failed to create tmp dir")
+		return "", err
 	}
 	defer os.RemoveAll(tmpDir)
 

pkg/gitlab_registry.go

@@ -0,0 +1,77 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"context"
+	"github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps"
+	"net/url"
+	"os"
+	"path"
+)
+
+type GitlabRegistryPackage struct {
+	Source *deps.GitlabRegistry
+}
+
+func NewGitlabRegistryPackage(source *deps.GitlabRegistry) Interface {
+	return &GitlabRegistryPackage{
+		Source: source,
+	}
+}
+
+func buildUrl(src *deps.GitlabRegistry, version string) url.URL {
+	host := "gitlab.com"
+	if src.Host != "" {
+		host = src.Host
+	}
+	filename := "package.tar.gz"
+	if src.Filename != "" {
+		filename = src.Filename
+	}
+
+	packageUrl := url.URL{
+		Scheme: "https",
+		Host:   host,
+	}
+	// this way is needed so that net/url does not double encode '/' to '%252F'
+	return *packageUrl.JoinPath(
+		"api/v4/projects",
+		url.PathEscape(src.Project),
+		"packages/generic",
+		src.Package,
+		version,
+		filename,
+	)
+}
+
+func (h *GitlabRegistryPackage) Install(ctx context.Context, name, dir, version string) (string, error) {
+	destPath := path.Join(dir, name)
+
+	packageUrl := buildUrl(h.Source, version)
+
+	tmpDir, err := CreateTempDir(name, dir, version)
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	err = DownloadAndUntarTo(tmpDir, packageUrl.String(), destPath)
+	if err != nil {
+		return "", err
+	}
+
+	return version, nil
+}

pkg/http.go

@@ -16,18 +16,9 @@ package pkg
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
 	"github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps"
-	"github.com/pkg/errors"
-	"io"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
-	"path/filepath"
-	"strings"
 )
 
 type HttpPackage struct {
@@ -40,66 +31,24 @@ func NewHttpPackage(source *deps.Http) Interface {
 	}
 }
 
-func downloadFile(filepath string, url string) error {
-	resp, err := http.Get(url)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("unexpected status code %d", resp.StatusCode)
-	}
-
-	defer resp.Body.Close()
-
-	// Create the file
-	out, err := os.Create(filepath)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	// Write the body to file
-	_, err = io.Copy(out, resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (h *HttpPackage) Install(ctx context.Context, name, dir, version string) (string, error) {
 	destPath := path.Join(dir, name)
 
-	pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1))))
+	err := os.Setenv("VERSION", version)
-	// using 16 bytes should be a good middle ground between length and collision resistance
-	tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16]))
-	if err != nil {
-		return "", errors.Wrap(err, "failed to create tmp dir")
-	}
-	defer os.RemoveAll(tmpDir)
-
-	err = os.Setenv("VERSION", version)
 	if err != nil {
 		return "", err
 	}
-	url := os.ExpandEnv(h.Source.Url)
+	packageUrl := os.ExpandEnv(h.Source.Url)
 
-	filename := filepath.Base(url)
+	tmpDir, err := CreateTempDir(name, dir, version)
-	err = downloadFile(path.Join(tmpDir, filename), url)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to download file")
+		return "", err
 	}
+	defer os.RemoveAll(tmpDir)
 
-	var ar *os.File
+	err = DownloadAndUntarTo(tmpDir, packageUrl, destPath)
-	ar, err = os.Open(path.Join(tmpDir, filename))
 	if err != nil {
-		return "", errors.Wrap(err, "failed to open downloaded archive")
+		return "", err
-	}
-	defer ar.Close()
-	err = GzipUntar(destPath, ar, "")
-	if err != nil {
-		return "", errors.Wrap(err, "failed to unpack downloaded archive")
 	}
 
 	return version, nil

pkg/packages.go

@@ -302,10 +302,12 @@ func download(d deps.Dependency, vendorDir, pathToParentModule string) (*deps.De
 		p = NewLocalPackage(&deps.Local{Directory: modulePath})
 	case d.Source.HttpSource != nil:
 		p = NewHttpPackage(d.Source.HttpSource)
+	case d.Source.GitlabRegistrySource != nil:
+		p = NewGitlabRegistryPackage(d.Source.GitlabRegistrySource)
 	}
 
 	if p == nil {
-		return nil, errors.New("either git, local or http source is required")
+		return nil, errors.New("either git, local, http or gitlab source is required")
 	}
 
 	version, err := p.Install(context.TODO(), d.Name(), vendorDir, d.Version)

pkg/utils.go

@@ -3,8 +3,16 @@ package pkg
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
 	"io"
+	"io/ioutil"
+	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -91,3 +99,64 @@ func GzipUntar(dst string, r io.Reader, subDir string) error {
 		}
 	}
 }
+
+func CreateTempDir(name, dir, version string) (string, error) {
+	pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1))))
+	// using 16 bytes should be a good middle ground between length and collision resistance
+	tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16]))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create tmp dir")
+	}
+
+	return tmpDir, nil
+}
+
+func DownloadFile(filepath string, url string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	color.Cyan("GET %s %d", url, resp.StatusCode)
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DownloadAndUntarTo(tmpDir, url, destPath string) error {
+	filename := filepath.Base(url)
+	err := DownloadFile(path.Join(tmpDir, filename), url)
+	if err != nil {
+		return errors.Wrap(err, "failed to download file")
+	}
+
+	var ar *os.File
+	ar, err = os.Open(path.Join(tmpDir, filename))
+	if err != nil {
+		return errors.Wrap(err, "failed to open downloaded archive")
+	}
+	defer ar.Close()
+	err = GzipUntar(destPath, ar, "")
+	if err != nil {
+		return errors.Wrap(err, "failed to unpack downloaded archive")
+	}
+	return nil
+}

spec/v1/deps/dependencies.go

@@ -14,6 +14,7 @@
 package deps
 
 import (
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -61,6 +62,7 @@ type Source struct {
 	GitSource            *Git            `json:"git,omitempty"`
 	LocalSource          *Local          `json:"local,omitempty"`
 	HttpSource           *Http           `json:"http,omitempty"`
+	GitlabRegistrySource *GitlabRegistry `json:"gitlab,omitempty"`
 }
 
 func (s Source) Name() string {
@@ -70,7 +72,13 @@ func (s Source) Name() string {
 	case s.LocalSource != nil:
 		return s.LegacyName()
 	case s.HttpSource != nil:
-		return s.LegacyName()
+		parsed, err := url.Parse(s.HttpSource.Url)
+		if err != nil {
+			return "http/" + s.LegacyName()
+		}
+		return parsed.Hostname() + "/" + s.LegacyName()
+	case s.GitlabRegistrySource != nil:
+		return "gitlab.com/" + s.LegacyName()
 	default:
 		return ""
 	}
@@ -96,6 +104,8 @@ func (s Source) LegacyName() string {
 			return strings.TrimSuffix(file, ".tar.gz")
 		}
 		return strings.Replace(file, filepath.Ext(file), "", 1)
+	case s.GitlabRegistrySource != nil:
+		return s.GitlabRegistrySource.Package
 	default:
 		return ""
 	}
@@ -144,3 +154,10 @@ func parseHttp(uri string) *Dependency {
 		Version: "",
 	}
 }
+
+type GitlabRegistry struct {
+	Project  string `json:"project"`
+	Package  string `json:"package"`
+	Host     string `json:"host,omitempty"`
+	Filename string `json:"filename,omitempty"`
+}