feat: rewrite install procedure

rewrites the installation of packages from scratch to solve several issues with
the existing implementation:

- does not need to choose between lockfile and jsonnetfile anymore. The
jsonnetfile what to be installed, while the lockfile also has versions and
checksums of all packages, even nested ones.
- the lockfile is regenerated on every run, preserving the locked values
- downloaded packages are hashed using sha256 to make sure we receive what we
expect. If files on the local disk are modified, they are downloaded again.

cmd/jb/install.go

@@ -15,12 +15,12 @@
 package main
 
 import (
-	"context"
 	"encoding/json"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/jsonnet-bundler/jsonnet-bundler/pkg"
@@ -28,92 +28,52 @@ import (
 	"github.com/jsonnet-bundler/jsonnet-bundler/spec"
 )
 
-func installCommand(dir, jsonnetHome string, uris ...string) int {
+func installCommand(dir, jsonnetHome string, uris []string) int {
 	if dir == "" {
 		dir = "."
 	}
 
-	filename, isLock, err := jsonnetfile.Choose(dir)
+	kingpin.FatalIfError(
-	if err != nil {
+		os.MkdirAll(filepath.Join(dir, "vendor", ".tmp"), os.ModePerm),
-		kingpin.Fatalf("failed to choose jsonnetfile: %v", err)
+		"creating vendor/ folder")
-		return 1
+
+	jsonnetFile, err := jsonnetfile.Load(filepath.Join(dir, jsonnetfile.File))
+	kingpin.FatalIfError(err, "failed to load jsonnetfile")
+
+	for _, u := range uris {
+		d := parseDependency(dir, u)
+		jsonnetFile.Dependencies = append(jsonnetFile.Dependencies, *d)
 	}
 
-	jsonnetFile, err := jsonnetfile.Load(filename)
+	lockFile, err := jsonnetfile.Load(filepath.Join(dir, jsonnetfile.LockFile))
-	if err != nil {
+	if !os.IsNotExist(err) {
-		kingpin.Fatalf("failed to load jsonnetfile: %v", err)
+		kingpin.FatalIfError(err, "failed to load lockfile")
-		return 1
 	}
 
-	if len(uris) > 0 {
+	locks := make(map[string]spec.Dependency)
-		for _, uri := range uris {
+	for _, d := range lockFile.Dependencies {
-			newDep := parseDependency(dir, uri)
+		locks[d.Name] = d
-			if newDep == nil {
-				kingpin.Errorf("ignoring unrecognized uri: %s", uri)
-				continue
-			}
-
-			oldDeps := jsonnetFile.Dependencies
-			newDeps := []spec.Dependency{}
-			oldDepReplaced := false
-			for _, d := range oldDeps {
-				if d.Name == newDep.Name {
-					newDeps = append(newDeps, *newDep)
-					oldDepReplaced = true
-				} else {
-					newDeps = append(newDeps, d)
-				}
-			}
-
-			if !oldDepReplaced {
-				newDeps = append(newDeps, *newDep)
-			}
-
-			jsonnetFile.Dependencies = newDeps
-		}
 	}
 
-	srcPath := filepath.Join(jsonnetHome)
+	locked, err := pkg.Ensure(jsonnetFile, jsonnetHome, locks)
-	err = os.MkdirAll(srcPath, os.ModePerm)
+	kingpin.FatalIfError(err, "failed to install packages")
-	if err != nil {
-		kingpin.Fatalf("failed to create jsonnet home path: %v", err)
-		return 3
-	}
 
-	lock, err := pkg.Install(context.TODO(), isLock, filename, jsonnetFile, jsonnetHome)
+	kingpin.FatalIfError(
-	if err != nil {
+		writeJSONFile(filepath.Join(dir, jsonnetfile.File), jsonnetFile),
-		kingpin.Fatalf("failed to install: %v", err)
+		"updating jsonnetfile.json")
-		return 3
+	kingpin.FatalIfError(
-	}
+		writeJSONFile(filepath.Join(dir, jsonnetfile.LockFile), spec.JsonnetFile{Dependencies: locked}),
-
+		"updating jsonnetfile.lock.json")
-	// If installing from lock file there is no need to write any files back.
-	if !isLock {
-		b, err := json.MarshalIndent(jsonnetFile, "", "    ")
-		if err != nil {
-			kingpin.Fatalf("failed to encode jsonnet file: %v", err)
-			return 3
-		}
-		b = append(b, []byte("\n")...)
-
-		err = ioutil.WriteFile(filepath.Join(dir, jsonnetfile.File), b, 0644)
-		if err != nil {
-			kingpin.Fatalf("failed to write jsonnet file: %v", err)
-			return 3
-		}
-
-		b, err = json.MarshalIndent(lock, "", "    ")
-		if err != nil {
-			kingpin.Fatalf("failed to encode jsonnet file: %v", err)
-			return 3
-		}
-		b = append(b, []byte("\n")...)
-
-		err = ioutil.WriteFile(filepath.Join(dir, jsonnetfile.LockFile), b, 0644)
-		if err != nil {
-			kingpin.Fatalf("failed to write lock file: %v", err)
-			return 3
-		}
-	}
 
 	return 0
 }
+
+func writeJSONFile(name string, d interface{}) error {
+	b, err := json.MarshalIndent(d, "", "  ")
+	if err != nil {
+		return errors.Wrap(err, "encoding json")
+	}
+	b = append(b, []byte("\n")...)
+
+	return ioutil.WriteFile(name, b, 0644)
+}

cmd/jb/main.go

@@ -83,11 +83,11 @@ func Main() int {
 	case initCmd.FullCommand():
 		return initCommand(workdir)
 	case installCmd.FullCommand():
-		return installCommand(workdir, cfg.JsonnetHome, *installCmdURIs...)
+		return installCommand(workdir, cfg.JsonnetHome, *installCmdURIs)
 	case updateCmd.FullCommand():
 		return updateCommand(cfg.JsonnetHome)
 	default:
-		installCommand(workdir, cfg.JsonnetHome)
+		installCommand(workdir, cfg.JsonnetHome, []string{})
 	}
 
 	return 0

pkg/jsonnetfile/jsonnetfile.go

@@ -60,7 +60,7 @@ func Load(filepath string) (spec.JsonnetFile, error) {
 
 	bytes, err := ioutil.ReadFile(filepath)
 	if err != nil {
-		return m, errors.Wrap(err, "failed to read file")
+		return m, err
 	}
 
 	if err := json.Unmarshal(bytes, &m); err != nil {

pkg/packages.go

@@ -16,12 +16,12 @@ package pkg
 
 import (
 	"context"
-	"fmt"
+	"crypto/sha256"
+	"encoding/base64"
+	"io"
 	"os"
-	"path"
 	"path/filepath"
 
-	"github.com/fatih/color"
 	"github.com/pkg/errors"
 
 	"github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile"
@@ -32,98 +32,111 @@ var (
 	VersionMismatch = errors.New("multiple colliding versions specified")
 )
 
-func Install(ctx context.Context, isLock bool, dependencySourceIdentifier string, m spec.JsonnetFile, dir string) (*spec.JsonnetFile, error) {
+func Ensure(want spec.JsonnetFile, vendorDir string, locks map[string]spec.Dependency) ([]spec.Dependency, error) {
-	lockfile := &spec.JsonnetFile{}
+	var list []spec.Dependency
-	for _, dep := range m.Dependencies {
+	for _, d := range want.Dependencies {
+		l, present := locks[d.Name]
 
-		tmp := filepath.Join(dir, ".tmp")
+		// already locked and the integrity is intact
-		err := os.MkdirAll(tmp, os.ModePerm)
+		if present && check(l, vendorDir) {
-		if err != nil {
+			list = append(list, l)
-			return nil, errors.Wrap(err, "failed to create general tmp dir")
-		}
-
-		var p Interface
-		if dep.Source.GitSource != nil {
-			p = NewGitPackage(dep.Source.GitSource)
-		}
-		if dep.Source.LocalSource != nil {
-			p = NewLocalPackage(dep.Source.LocalSource)
-		}
-
-		lockVersion, err := p.Install(ctx, dep.Name, dir, dep.Version)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to install package")
-		}
-
-		color.Green(">>> Installed %s version %s\n", dep.Name, dep.Version)
-
-		destPath := path.Join(dir, dep.Name)
-
-		lockfile.Dependencies, err = insertDependency(lockfile.Dependencies, spec.Dependency{
-			Name:      dep.Name,
-			Source:    dep.Source,
-			Version:   lockVersion,
-			DepSource: dependencySourceIdentifier,
-		})
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to insert dependency to lock dependencies")
-		}
-
-		// If dependencies are being installed from a lock file, the transitive
-		// dependencies are not questioned, the locked dependencies are just
-		// installed.
-		if isLock {
 			continue
 		}
 
-		filepath, isLock, err := jsonnetfile.Choose(destPath)
+		// either not present or not intact: download again
+		dir := filepath.Join(vendorDir, d.Name)
+		os.RemoveAll(dir)
+
+		locked, err := download(d, vendorDir)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "downloading")
 		}
-		depsDeps, err := jsonnetfile.Load(filepath)
+		list = append(list, *locked)
-		// It is ok for dependencies not to have a JsonnetFile, it just means
+	}
-		// they do not have transitive dependencies of their own.
+
-		if err != nil && !os.IsNotExist(err) {
+	for _, d := range list {
+		f, err := jsonnetfile.Load(filepath.Join(vendorDir, d.Name, jsonnetfile.File))
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
 			return nil, err
 		}
 
-		depsInstalledByDependency, err := Install(ctx, isLock, filepath, depsDeps, dir)
+		nested, err := Ensure(f, vendorDir, locks)
 		if err != nil {
 			return nil, err
 		}
 
-		for _, d := range depsInstalledByDependency.Dependencies {
+		list = append(list, nested...)
-			lockfile.Dependencies, err = insertDependency(lockfile.Dependencies, d)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to insert dependency to lock dependencies")
-			}
-		}
 	}
 
-	return lockfile, nil
+	return list, nil
 }
 
-func insertDependency(deps []spec.Dependency, newDep spec.Dependency) ([]spec.Dependency, error) {
+func download(d spec.Dependency, vendorDir string) (*spec.Dependency, error) {
-	if len(deps) == 0 {
+	var p Interface
-		return []spec.Dependency{newDep}, nil
+	switch {
+	case d.Source.GitSource != nil:
+		p = NewGitPackage(d.Source.GitSource)
+	case d.Source.LocalSource != nil:
+		p = NewLocalPackage(d.Source.LocalSource)
 	}
 
-	res := []spec.Dependency{}
+	if p == nil {
-	newDepPreviouslyPresent := false
+		return nil, errors.New("either git or local source is required")
-	for _, d := range deps {
-		if d.Name == newDep.Name {
-			if d.Version != newDep.Version {
-				return nil, fmt.Errorf("multiple colliding versions specified for %s: %s (from %s) and %s (from %s)", d.Name, d.Version, d.DepSource, newDep.Version, newDep.DepSource)
-			}
-			res = append(res, d)
-			newDepPreviouslyPresent = true
-		} else {
-			res = append(res, d)
-		}
-	}
-	if !newDepPreviouslyPresent {
-		res = append(res, newDep)
 	}
 
-	return res, nil
+	version, err := p.Install(context.TODO(), d.Name, vendorDir, d.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	sum := hashDir(filepath.Join(vendorDir, d.Name))
+
+	return &spec.Dependency{
+		Name:    d.Name,
+		Source:  d.Source,
+		Version: version,
+		Sum:     sum,
+	}, nil
+}
+
+func check(d spec.Dependency, vendorDir string) bool {
+	if d.Sum == "" {
+		// no sum available, need to download
+		return false
+	}
+
+	dir := filepath.Join(vendorDir, d.Name)
+	sum := hashDir(dir)
+	return d.Sum == sum
+}
+
+func hashDir(dir string) string {
+	hasher := sha256.New()
+
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		if _, err := io.Copy(hasher, f); err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	return base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 }

spec/spec.go

@@ -22,6 +22,7 @@ type Dependency struct {
 	Name      string `json:"name"`
 	Source    Source `json:"source"`
 	Version   string `json:"version"`
+	Sum       string `json:"sum,omitempty"`
 	DepSource string `json:"-"`
 }