diff --git a/flake.nix b/flake.nix index 35072bd..b87a18e 100644 --- a/flake.nix +++ b/flake.nix @@ -4,7 +4,7 @@ outputs = { self, nixpkgs, systems }: let - version = "0.5.1"; + version = "0.5.2"; forEachSystem = nixpkgs.lib.genAttrs (import systems); nixpkgsFor = forEachSystem (system: import nixpkgs { inherit system; }); in { diff --git a/pkg/git.go b/pkg/git.go index 0a4f9c6..93551a9 100644 --- a/pkg/git.go +++ b/pkg/git.go @@ -17,8 +17,6 @@ package pkg import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "fmt" "io" "io/ioutil" @@ -97,11 +95,9 @@ func remoteResolveRef(ctx context.Context, remote string, ref string) (string, e func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (string, error) { destPath := path.Join(dir, name) - pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1)))) - // using 16 bytes should be a good middle ground between length and collision resistance - tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16])) + tmpDir, err := CreateTempDir(name, dir, version) if err != nil { - return "", errors.Wrap(err, "failed to create tmp dir") + return "", err } defer os.RemoveAll(tmpDir) diff --git a/pkg/gitlab_registry.go b/pkg/gitlab_registry.go new file mode 100644 index 0000000..98d099a --- /dev/null +++ b/pkg/gitlab_registry.go @@ -0,0 +1,77 @@ +// Copyright 2018 jsonnet-bundler authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkg + +import ( + "context" + "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps" + "net/url" + "os" + "path" +) + +type GitlabRegistryPackage struct { + Source *deps.GitlabRegistry +} + +func NewGitlabRegistryPackage(source *deps.GitlabRegistry) Interface { + return &GitlabRegistryPackage{ + Source: source, + } +} + +func buildUrl(src *deps.GitlabRegistry, version string) url.URL { + host := "gitlab.com" + if src.Host != "" { + host = src.Host + } + filename := "package.tar.gz" + if src.Filename != "" { + filename = src.Filename + } + + packageUrl := url.URL{ + Scheme: "https", + Host: host, + } + // this way is needed so that net/url does not double encode '/' to '%252F' + return *packageUrl.JoinPath( + "api/v4/projects", + url.PathEscape(src.Project), + "packages/generic", + src.Package, + version, + filename, + ) +} + +func (h *GitlabRegistryPackage) Install(ctx context.Context, name, dir, version string) (string, error) { + destPath := path.Join(dir, name) + + packageUrl := buildUrl(h.Source, version) + + tmpDir, err := CreateTempDir(name, dir, version) + if err != nil { + return "", err + } + defer os.RemoveAll(tmpDir) + + err = DownloadAndUntarTo(tmpDir, packageUrl.String(), destPath) + if err != nil { + return "", err + } + + return version, nil +} diff --git a/pkg/http.go b/pkg/http.go index 78190a2..ce7de25 100644 --- a/pkg/http.go +++ b/pkg/http.go @@ -16,18 +16,9 @@ package pkg import ( "context" - "crypto/sha256" - "encoding/hex" - "fmt" "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps" - "github.com/pkg/errors" - "io" - "io/ioutil" - "net/http" "os" "path" - "path/filepath" - "strings" ) type HttpPackage struct { @@ -40,66 +31,24 @@ func NewHttpPackage(source *deps.Http) Interface { } } -func downloadFile(filepath string, url string) error { - resp, err := http.Get(url) - if err != nil { - return err - } - - if resp.StatusCode != 200 { - return fmt.Errorf("unexpected status code %d", resp.StatusCode) - } - - defer resp.Body.Close() - - // Create the file - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - - // Write the body to file - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - - return nil -} - func (h *HttpPackage) Install(ctx context.Context, name, dir, version string) (string, error) { destPath := path.Join(dir, name) - pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1)))) - // using 16 bytes should be a good middle ground between length and collision resistance - tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16])) - if err != nil { - return "", errors.Wrap(err, "failed to create tmp dir") - } - defer os.RemoveAll(tmpDir) - - err = os.Setenv("VERSION", version) + err := os.Setenv("VERSION", version) if err != nil { return "", err } - url := os.ExpandEnv(h.Source.Url) + packageUrl := os.ExpandEnv(h.Source.Url) - filename := filepath.Base(url) - err = downloadFile(path.Join(tmpDir, filename), url) + tmpDir, err := CreateTempDir(name, dir, version) if err != nil { - return "", errors.Wrap(err, "failed to download file") + return "", err } + defer os.RemoveAll(tmpDir) - var ar *os.File - ar, err = os.Open(path.Join(tmpDir, filename)) + err = DownloadAndUntarTo(tmpDir, packageUrl, destPath) if err != nil { - return "", errors.Wrap(err, "failed to open downloaded archive") - } - defer ar.Close() - err = GzipUntar(destPath, ar, "") - if err != nil { - return "", errors.Wrap(err, "failed to unpack downloaded archive") + return "", err } return version, nil diff --git a/pkg/packages.go b/pkg/packages.go index f529aba..5f20151 100644 --- a/pkg/packages.go +++ b/pkg/packages.go @@ -302,10 +302,12 @@ func download(d deps.Dependency, vendorDir, pathToParentModule string) (*deps.De p = NewLocalPackage(&deps.Local{Directory: modulePath}) case d.Source.HttpSource != nil: p = NewHttpPackage(d.Source.HttpSource) + case d.Source.GitlabRegistrySource != nil: + p = NewGitlabRegistryPackage(d.Source.GitlabRegistrySource) } if p == nil { - return nil, errors.New("either git, local or http source is required") + return nil, errors.New("either git, local, http or gitlab source is required") } version, err := p.Install(context.TODO(), d.Name(), vendorDir, d.Version) diff --git a/pkg/utils.go b/pkg/utils.go index b1fc3bb..e87f47e 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -3,8 +3,16 @@ package pkg import ( "archive/tar" "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "github.com/fatih/color" + "github.com/pkg/errors" "io" + "io/ioutil" + "net/http" "os" + "path" "path/filepath" "strings" ) @@ -91,3 +99,64 @@ func GzipUntar(dst string, r io.Reader, subDir string) error { } } } + +func CreateTempDir(name, dir, version string) (string, error) { + pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1)))) + // using 16 bytes should be a good middle ground between length and collision resistance + tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16])) + if err != nil { + return "", errors.Wrap(err, "failed to create tmp dir") + } + + return tmpDir, nil +} + +func DownloadFile(filepath string, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + + color.Cyan("GET %s %d", url, resp.StatusCode) + + if resp.StatusCode != 200 { + return fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + defer resp.Body.Close() + + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} + +func DownloadAndUntarTo(tmpDir, url, destPath string) error { + filename := filepath.Base(url) + err := DownloadFile(path.Join(tmpDir, filename), url) + if err != nil { + return errors.Wrap(err, "failed to download file") + } + + var ar *os.File + ar, err = os.Open(path.Join(tmpDir, filename)) + if err != nil { + return errors.Wrap(err, "failed to open downloaded archive") + } + defer ar.Close() + err = GzipUntar(destPath, ar, "") + if err != nil { + return errors.Wrap(err, "failed to unpack downloaded archive") + } + return nil +} diff --git a/spec/v1/deps/dependencies.go b/spec/v1/deps/dependencies.go index 10655cf..a8d6243 100644 --- a/spec/v1/deps/dependencies.go +++ b/spec/v1/deps/dependencies.go @@ -14,6 +14,7 @@ package deps import ( + "net/url" "os" "path/filepath" "strings" @@ -58,9 +59,10 @@ func (d Dependency) LegacyName() string { } type Source struct { - GitSource *Git `json:"git,omitempty"` - LocalSource *Local `json:"local,omitempty"` - HttpSource *Http `json:"http,omitempty"` + GitSource *Git `json:"git,omitempty"` + LocalSource *Local `json:"local,omitempty"` + HttpSource *Http `json:"http,omitempty"` + GitlabRegistrySource *GitlabRegistry `json:"gitlab,omitempty"` } func (s Source) Name() string { @@ -70,7 +72,13 @@ func (s Source) Name() string { case s.LocalSource != nil: return s.LegacyName() case s.HttpSource != nil: - return s.LegacyName() + parsed, err := url.Parse(s.HttpSource.Url) + if err != nil { + return "http/" + s.LegacyName() + } + return parsed.Hostname() + "/" + s.LegacyName() + case s.GitlabRegistrySource != nil: + return "gitlab.com/" + s.LegacyName() default: return "" } @@ -96,6 +104,8 @@ func (s Source) LegacyName() string { return strings.TrimSuffix(file, ".tar.gz") } return strings.Replace(file, filepath.Ext(file), "", 1) + case s.GitlabRegistrySource != nil: + return s.GitlabRegistrySource.Package default: return "" } @@ -144,3 +154,10 @@ func parseHttp(uri string) *Dependency { Version: "", } } + +type GitlabRegistry struct { + Project string `json:"project"` + Package string `json:"package"` + Host string `json:"host,omitempty"` + Filename string `json:"filename,omitempty"` +}