From 428608f190f36e6be72e26e8f16fdc3a3e965592 Mon Sep 17 00:00:00 2001 From: Technofab Date: Tue, 29 Nov 2022 16:07:42 +0100 Subject: [PATCH] feat: add http source with support for environment variables --- pkg/git.go | 87 +-------------------------------------- pkg/http.go | 106 ++++++++++++++++++++++++++++++++++++++++++++++++ pkg/packages.go | 4 +- pkg/utils.go | 93 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 87 deletions(-) create mode 100644 pkg/http.go create mode 100644 pkg/utils.go diff --git a/pkg/git.go b/pkg/git.go index afe9cfe..0a4f9c6 100644 --- a/pkg/git.go +++ b/pkg/git.go @@ -15,9 +15,7 @@ package pkg import ( - "archive/tar" "bytes" - "compress/gzip" "context" "crypto/sha256" "encoding/hex" @@ -81,89 +79,6 @@ func downloadGitHubArchive(filepath string, url string) error { return nil } -func gzipUntar(dst string, r io.Reader, subDir string) error { - gzr, err := gzip.NewReader(r) - if err != nil { - return err - } - defer gzr.Close() - - tr := tar.NewReader(gzr) - - for { - header, err := tr.Next() - switch { - case err == io.EOF: - return nil - - case err != nil: - return err - - case header == nil: - continue - } - - // strip the two first components of the path - parts := strings.SplitAfterN(header.Name, "/", 2) - if len(parts) < 2 { - continue - } - suffix := parts[1] - prefix := dst - - // reconstruct the target parh for the archive entry - target := filepath.Join(prefix, suffix) - - // if subdir is provided and target is not under it, skip it - subDirPath := filepath.Join(prefix, subDir) - if subDir != "" && !strings.HasPrefix(target, subDirPath) { - continue - } - - // check the file type - switch header.Typeflag { - - // create directories as needed - case tar.TypeDir: - if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { - return err - } - - case tar.TypeReg: - if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil { - return err - } - - err := func() error { - f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) - if err != nil { - return err - } - defer f.Close() - - // copy over contents - if _, err := io.Copy(f, tr); err != nil { - return err - } - return nil - }() - - if err != nil { - return err - } - - case tar.TypeSymlink: - if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil { - return err - } - - if err := os.Symlink(header.Linkname, target); err != nil { - return err - } - } - } -} - func remoteResolveRef(ctx context.Context, remote string, ref string) (string, error) { b := &bytes.Buffer{} cmd := exec.CommandContext(ctx, "git", "ls-remote", "--heads", "--tags", "--refs", "--quiet", remote, ref) @@ -217,7 +132,7 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st if err == nil { // Extract the sub-directory (if any) from the archive // If none specified, the entire archive is unpacked - err = gzipUntar(tmpDir, ar, p.Source.Subdir) + err = GzipUntar(tmpDir, ar, p.Source.Subdir) // Move the extracted directory to its final destination if err == nil { diff --git a/pkg/http.go b/pkg/http.go new file mode 100644 index 0000000..78190a2 --- /dev/null +++ b/pkg/http.go @@ -0,0 +1,106 @@ +// Copyright 2018 jsonnet-bundler authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkg + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps" + "github.com/pkg/errors" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +type HttpPackage struct { + Source *deps.Http +} + +func NewHttpPackage(source *deps.Http) Interface { + return &HttpPackage{ + Source: source, + } +} + +func downloadFile(filepath string, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + return fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + defer resp.Body.Close() + + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} + +func (h *HttpPackage) Install(ctx context.Context, name, dir, version string) (string, error) { + destPath := path.Join(dir, name) + + pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1)))) + // using 16 bytes should be a good middle ground between length and collision resistance + tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16])) + if err != nil { + return "", errors.Wrap(err, "failed to create tmp dir") + } + defer os.RemoveAll(tmpDir) + + err = os.Setenv("VERSION", version) + if err != nil { + return "", err + } + url := os.ExpandEnv(h.Source.Url) + + filename := filepath.Base(url) + err = downloadFile(path.Join(tmpDir, filename), url) + if err != nil { + return "", errors.Wrap(err, "failed to download file") + } + + var ar *os.File + ar, err = os.Open(path.Join(tmpDir, filename)) + if err != nil { + return "", errors.Wrap(err, "failed to open downloaded archive") + } + defer ar.Close() + err = GzipUntar(destPath, ar, "") + if err != nil { + return "", errors.Wrap(err, "failed to unpack downloaded archive") + } + + return version, nil +} diff --git a/pkg/packages.go b/pkg/packages.go index ab14941..f529aba 100644 --- a/pkg/packages.go +++ b/pkg/packages.go @@ -300,10 +300,12 @@ func download(d deps.Dependency, vendorDir, pathToParentModule string) (*deps.De } p = NewLocalPackage(&deps.Local{Directory: modulePath}) + case d.Source.HttpSource != nil: + p = NewHttpPackage(d.Source.HttpSource) } if p == nil { - return nil, errors.New("either git or local source is required") + return nil, errors.New("either git, local or http source is required") } version, err := p.Install(context.TODO(), d.Name(), vendorDir, d.Version) diff --git a/pkg/utils.go b/pkg/utils.go new file mode 100644 index 0000000..b1fc3bb --- /dev/null +++ b/pkg/utils.go @@ -0,0 +1,93 @@ +package pkg + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" +) + +func GzipUntar(dst string, r io.Reader, subDir string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + switch { + case err == io.EOF: + return nil + + case err != nil: + return err + + case header == nil: + continue + } + + // strip the two first components of the path + parts := strings.SplitAfterN(header.Name, "/", 2) + if len(parts) < 2 { + continue + } + suffix := parts[1] + prefix := dst + + // reconstruct the target parh for the archive entry + target := filepath.Join(prefix, suffix) + + // if subdir is provided and target is not under it, skip it + subDirPath := filepath.Join(prefix, subDir) + if subDir != "" && !strings.HasPrefix(target, subDirPath) { + continue + } + + // check the file type + switch header.Typeflag { + + // create directories as needed + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { + return err + } + + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil { + return err + } + + err := func() error { + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + defer f.Close() + + // copy over contents + if _, err := io.Copy(f, tr); err != nil { + return err + } + return nil + }() + + if err != nil { + return err + } + + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil { + return err + } + + if err := os.Symlink(header.Linkname, target); err != nil { + return err + } + } + } +}