feat: add http source with support for environment variables

pkg/git.go

@@ -15,9 +15,7 @@
 package pkg
 
 import (
-	"archive/tar"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -81,89 +79,6 @@ func downloadGitHubArchive(filepath string, url string) error {
 	return nil
 }
 
-func gzipUntar(dst string, r io.Reader, subDir string) error {
-	gzr, err := gzip.NewReader(r)
-	if err != nil {
-		return err
-	}
-	defer gzr.Close()
-
-	tr := tar.NewReader(gzr)
-
-	for {
-		header, err := tr.Next()
-		switch {
-		case err == io.EOF:
-			return nil
-
-		case err != nil:
-			return err
-
-		case header == nil:
-			continue
-		}
-
-		// strip the two first components of the path
-		parts := strings.SplitAfterN(header.Name, "/", 2)
-		if len(parts) < 2 {
-			continue
-		}
-		suffix := parts[1]
-		prefix := dst
-
-		// reconstruct the target parh for the archive entry
-		target := filepath.Join(prefix, suffix)
-
-		// if subdir is provided and target is not under it, skip it
-		subDirPath := filepath.Join(prefix, subDir)
-		if subDir != "" && !strings.HasPrefix(target, subDirPath) {
-			continue
-		}
-
-		// check the file type
-		switch header.Typeflag {
-
-		// create directories as needed
-		case tar.TypeDir:
-			if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil {
-				return err
-			}
-
-		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil {
-				return err
-			}
-
-			err := func() error {
-				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-				if err != nil {
-					return err
-				}
-				defer f.Close()
-
-				// copy over contents
-				if _, err := io.Copy(f, tr); err != nil {
-					return err
-				}
-				return nil
-			}()
-
-			if err != nil {
-				return err
-			}
-
-		case tar.TypeSymlink:
-			if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil {
-				return err
-			}
-
-			if err := os.Symlink(header.Linkname, target); err != nil {
-				return err
-			}
-		}
-	}
-}
-
 func remoteResolveRef(ctx context.Context, remote string, ref string) (string, error) {
 	b := &bytes.Buffer{}
 	cmd := exec.CommandContext(ctx, "git", "ls-remote", "--heads", "--tags", "--refs", "--quiet", remote, ref)
@@ -217,7 +132,7 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st
 			if err == nil {
 				// Extract the sub-directory (if any) from the archive
 				// If none specified, the entire archive is unpacked
-				err = gzipUntar(tmpDir, ar, p.Source.Subdir)
+				err = GzipUntar(tmpDir, ar, p.Source.Subdir)
 
 				// Move the extracted directory to its final destination
 				if err == nil {

pkg/http.go

@@ -0,0 +1,106 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps"
+	"github.com/pkg/errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type HttpPackage struct {
+	Source *deps.Http
+}
+
+func NewHttpPackage(source *deps.Http) Interface {
+	return &HttpPackage{
+		Source: source,
+	}
+}
+
+func downloadFile(filepath string, url string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *HttpPackage) Install(ctx context.Context, name, dir, version string) (string, error) {
+	destPath := path.Join(dir, name)
+
+	pkgh := sha256.Sum256([]byte(fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), strings.Replace(version, "/", "-", -1))))
+	// using 16 bytes should be a good middle ground between length and collision resistance
+	tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), hex.EncodeToString(pkgh[:16]))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create tmp dir")
+	}
+	defer os.RemoveAll(tmpDir)
+
+	err = os.Setenv("VERSION", version)
+	if err != nil {
+		return "", err
+	}
+	url := os.ExpandEnv(h.Source.Url)
+
+	filename := filepath.Base(url)
+	err = downloadFile(path.Join(tmpDir, filename), url)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to download file")
+	}
+
+	var ar *os.File
+	ar, err = os.Open(path.Join(tmpDir, filename))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to open downloaded archive")
+	}
+	defer ar.Close()
+	err = GzipUntar(destPath, ar, "")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to unpack downloaded archive")
+	}
+
+	return version, nil
+}

pkg/packages.go

@@ -300,10 +300,12 @@ func download(d deps.Dependency, vendorDir, pathToParentModule string) (*deps.De
 		}
 
 		p = NewLocalPackage(&deps.Local{Directory: modulePath})
+	case d.Source.HttpSource != nil:
+		p = NewHttpPackage(d.Source.HttpSource)
 	}
 
 	if p == nil {
-		return nil, errors.New("either git or local source is required")
+		return nil, errors.New("either git, local or http source is required")
 	}
 
 	version, err := p.Install(context.TODO(), d.Name(), vendorDir, d.Version)

pkg/utils.go

@@ -0,0 +1,93 @@
+package pkg
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func GzipUntar(dst string, r io.Reader, subDir string) error {
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+		switch {
+		case err == io.EOF:
+			return nil
+
+		case err != nil:
+			return err
+
+		case header == nil:
+			continue
+		}
+
+		// strip the two first components of the path
+		parts := strings.SplitAfterN(header.Name, "/", 2)
+		if len(parts) < 2 {
+			continue
+		}
+		suffix := parts[1]
+		prefix := dst
+
+		// reconstruct the target parh for the archive entry
+		target := filepath.Join(prefix, suffix)
+
+		// if subdir is provided and target is not under it, skip it
+		subDirPath := filepath.Join(prefix, subDir)
+		if subDir != "" && !strings.HasPrefix(target, subDirPath) {
+			continue
+		}
+
+		// check the file type
+		switch header.Typeflag {
+
+		// create directories as needed
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil {
+				return err
+			}
+
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil {
+				return err
+			}
+
+			err := func() error {
+				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+				if err != nil {
+					return err
+				}
+				defer f.Close()
+
+				// copy over contents
+				if _, err := io.Copy(f, tr); err != nil {
+					return err
+				}
+				return nil
+			}()
+
+			if err != nil {
+				return err
+			}
+
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(target), os.ModePerm); err != nil {
+				return err
+			}
+
+			if err := os.Symlink(header.Linkname, target); err != nil {
+				return err
+			}
+		}
+	}
+}