diff --git a/cmd/jb/install.go b/cmd/jb/install.go index 548134b..7e13ba9 100644 --- a/cmd/jb/install.go +++ b/cmd/jb/install.go @@ -15,12 +15,12 @@ package main import ( - "context" "encoding/json" "io/ioutil" "os" "path/filepath" + "github.com/pkg/errors" kingpin "gopkg.in/alecthomas/kingpin.v2" "github.com/jsonnet-bundler/jsonnet-bundler/pkg" @@ -28,92 +28,52 @@ import ( "github.com/jsonnet-bundler/jsonnet-bundler/spec" ) -func installCommand(dir, jsonnetHome string, uris ...string) int { +func installCommand(dir, jsonnetHome string, uris []string) int { if dir == "" { dir = "." } - filename, isLock, err := jsonnetfile.Choose(dir) - if err != nil { - kingpin.Fatalf("failed to choose jsonnetfile: %v", err) - return 1 + kingpin.FatalIfError( + os.MkdirAll(filepath.Join(dir, "vendor", ".tmp"), os.ModePerm), + "creating vendor/ folder") + + jsonnetFile, err := jsonnetfile.Load(filepath.Join(dir, jsonnetfile.File)) + kingpin.FatalIfError(err, "failed to load jsonnetfile") + + for _, u := range uris { + d := parseDependency(dir, u) + jsonnetFile.Dependencies = append(jsonnetFile.Dependencies, *d) } - jsonnetFile, err := jsonnetfile.Load(filename) - if err != nil { - kingpin.Fatalf("failed to load jsonnetfile: %v", err) - return 1 + lockFile, err := jsonnetfile.Load(filepath.Join(dir, jsonnetfile.LockFile)) + if !os.IsNotExist(err) { + kingpin.FatalIfError(err, "failed to load lockfile") } - if len(uris) > 0 { - for _, uri := range uris { - newDep := parseDependency(dir, uri) - if newDep == nil { - kingpin.Errorf("ignoring unrecognized uri: %s", uri) - continue - } - - oldDeps := jsonnetFile.Dependencies - newDeps := []spec.Dependency{} - oldDepReplaced := false - for _, d := range oldDeps { - if d.Name == newDep.Name { - newDeps = append(newDeps, *newDep) - oldDepReplaced = true - } else { - newDeps = append(newDeps, d) - } - } - - if !oldDepReplaced { - newDeps = append(newDeps, *newDep) - } - - jsonnetFile.Dependencies = newDeps - } + locks := make(map[string]spec.Dependency) + for _, d := range lockFile.Dependencies { + locks[d.Name] = d } - srcPath := filepath.Join(jsonnetHome) - err = os.MkdirAll(srcPath, os.ModePerm) - if err != nil { - kingpin.Fatalf("failed to create jsonnet home path: %v", err) - return 3 - } + locked, err := pkg.Ensure(jsonnetFile, jsonnetHome, locks) + kingpin.FatalIfError(err, "failed to install packages") - lock, err := pkg.Install(context.TODO(), isLock, filename, jsonnetFile, jsonnetHome) - if err != nil { - kingpin.Fatalf("failed to install: %v", err) - return 3 - } - - // If installing from lock file there is no need to write any files back. - if !isLock { - b, err := json.MarshalIndent(jsonnetFile, "", " ") - if err != nil { - kingpin.Fatalf("failed to encode jsonnet file: %v", err) - return 3 - } - b = append(b, []byte("\n")...) - - err = ioutil.WriteFile(filepath.Join(dir, jsonnetfile.File), b, 0644) - if err != nil { - kingpin.Fatalf("failed to write jsonnet file: %v", err) - return 3 - } - - b, err = json.MarshalIndent(lock, "", " ") - if err != nil { - kingpin.Fatalf("failed to encode jsonnet file: %v", err) - return 3 - } - b = append(b, []byte("\n")...) - - err = ioutil.WriteFile(filepath.Join(dir, jsonnetfile.LockFile), b, 0644) - if err != nil { - kingpin.Fatalf("failed to write lock file: %v", err) - return 3 - } - } + kingpin.FatalIfError( + writeJSONFile(filepath.Join(dir, jsonnetfile.File), jsonnetFile), + "updating jsonnetfile.json") + kingpin.FatalIfError( + writeJSONFile(filepath.Join(dir, jsonnetfile.LockFile), spec.JsonnetFile{Dependencies: locked}), + "updating jsonnetfile.lock.json") return 0 } + +func writeJSONFile(name string, d interface{}) error { + b, err := json.MarshalIndent(d, "", " ") + if err != nil { + return errors.Wrap(err, "encoding json") + } + b = append(b, []byte("\n")...) + + return ioutil.WriteFile(name, b, 0644) +} diff --git a/cmd/jb/main.go b/cmd/jb/main.go index ba12762..782aa78 100644 --- a/cmd/jb/main.go +++ b/cmd/jb/main.go @@ -83,11 +83,11 @@ func Main() int { case initCmd.FullCommand(): return initCommand(workdir) case installCmd.FullCommand(): - return installCommand(workdir, cfg.JsonnetHome, *installCmdURIs...) + return installCommand(workdir, cfg.JsonnetHome, *installCmdURIs) case updateCmd.FullCommand(): return updateCommand(cfg.JsonnetHome) default: - installCommand(workdir, cfg.JsonnetHome) + installCommand(workdir, cfg.JsonnetHome, []string{}) } return 0 diff --git a/pkg/jsonnetfile/jsonnetfile.go b/pkg/jsonnetfile/jsonnetfile.go index b82c25f..78e6a48 100644 --- a/pkg/jsonnetfile/jsonnetfile.go +++ b/pkg/jsonnetfile/jsonnetfile.go @@ -60,7 +60,7 @@ func Load(filepath string) (spec.JsonnetFile, error) { bytes, err := ioutil.ReadFile(filepath) if err != nil { - return m, errors.Wrap(err, "failed to read file") + return m, err } if err := json.Unmarshal(bytes, &m); err != nil { diff --git a/pkg/packages.go b/pkg/packages.go index 2b4650e..35ff19f 100644 --- a/pkg/packages.go +++ b/pkg/packages.go @@ -16,12 +16,12 @@ package pkg import ( "context" - "fmt" + "crypto/sha256" + "encoding/base64" + "io" "os" - "path" "path/filepath" - "github.com/fatih/color" "github.com/pkg/errors" "github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile" @@ -32,98 +32,111 @@ var ( VersionMismatch = errors.New("multiple colliding versions specified") ) -func Install(ctx context.Context, isLock bool, dependencySourceIdentifier string, m spec.JsonnetFile, dir string) (*spec.JsonnetFile, error) { - lockfile := &spec.JsonnetFile{} - for _, dep := range m.Dependencies { +func Ensure(want spec.JsonnetFile, vendorDir string, locks map[string]spec.Dependency) ([]spec.Dependency, error) { + var list []spec.Dependency + for _, d := range want.Dependencies { + l, present := locks[d.Name] - tmp := filepath.Join(dir, ".tmp") - err := os.MkdirAll(tmp, os.ModePerm) - if err != nil { - return nil, errors.Wrap(err, "failed to create general tmp dir") - } - - var p Interface - if dep.Source.GitSource != nil { - p = NewGitPackage(dep.Source.GitSource) - } - if dep.Source.LocalSource != nil { - p = NewLocalPackage(dep.Source.LocalSource) - } - - lockVersion, err := p.Install(ctx, dep.Name, dir, dep.Version) - if err != nil { - return nil, errors.Wrap(err, "failed to install package") - } - - color.Green(">>> Installed %s version %s\n", dep.Name, dep.Version) - - destPath := path.Join(dir, dep.Name) - - lockfile.Dependencies, err = insertDependency(lockfile.Dependencies, spec.Dependency{ - Name: dep.Name, - Source: dep.Source, - Version: lockVersion, - DepSource: dependencySourceIdentifier, - }) - if err != nil { - return nil, errors.Wrap(err, "failed to insert dependency to lock dependencies") - } - - // If dependencies are being installed from a lock file, the transitive - // dependencies are not questioned, the locked dependencies are just - // installed. - if isLock { + // already locked and the integrity is intact + if present && check(l, vendorDir) { + list = append(list, l) continue } - filepath, isLock, err := jsonnetfile.Choose(destPath) + // either not present or not intact: download again + dir := filepath.Join(vendorDir, d.Name) + os.RemoveAll(dir) + + locked, err := download(d, vendorDir) if err != nil { - return nil, err + return nil, errors.Wrap(err, "downloading") } - depsDeps, err := jsonnetfile.Load(filepath) - // It is ok for dependencies not to have a JsonnetFile, it just means - // they do not have transitive dependencies of their own. - if err != nil && !os.IsNotExist(err) { + list = append(list, *locked) + } + + for _, d := range list { + f, err := jsonnetfile.Load(filepath.Join(vendorDir, d.Name, jsonnetfile.File)) + if err != nil { + if os.IsNotExist(err) { + continue + } return nil, err } - depsInstalledByDependency, err := Install(ctx, isLock, filepath, depsDeps, dir) + nested, err := Ensure(f, vendorDir, locks) if err != nil { return nil, err } - for _, d := range depsInstalledByDependency.Dependencies { - lockfile.Dependencies, err = insertDependency(lockfile.Dependencies, d) - if err != nil { - return nil, errors.Wrap(err, "failed to insert dependency to lock dependencies") - } - } + list = append(list, nested...) } - return lockfile, nil + return list, nil } -func insertDependency(deps []spec.Dependency, newDep spec.Dependency) ([]spec.Dependency, error) { - if len(deps) == 0 { - return []spec.Dependency{newDep}, nil +func download(d spec.Dependency, vendorDir string) (*spec.Dependency, error) { + var p Interface + switch { + case d.Source.GitSource != nil: + p = NewGitPackage(d.Source.GitSource) + case d.Source.LocalSource != nil: + p = NewLocalPackage(d.Source.LocalSource) } - res := []spec.Dependency{} - newDepPreviouslyPresent := false - for _, d := range deps { - if d.Name == newDep.Name { - if d.Version != newDep.Version { - return nil, fmt.Errorf("multiple colliding versions specified for %s: %s (from %s) and %s (from %s)", d.Name, d.Version, d.DepSource, newDep.Version, newDep.DepSource) - } - res = append(res, d) - newDepPreviouslyPresent = true - } else { - res = append(res, d) - } - } - if !newDepPreviouslyPresent { - res = append(res, newDep) + if p == nil { + return nil, errors.New("either git or local source is required") } - return res, nil + version, err := p.Install(context.TODO(), d.Name, vendorDir, d.Version) + if err != nil { + return nil, err + } + + sum := hashDir(filepath.Join(vendorDir, d.Name)) + + return &spec.Dependency{ + Name: d.Name, + Source: d.Source, + Version: version, + Sum: sum, + }, nil +} + +func check(d spec.Dependency, vendorDir string) bool { + if d.Sum == "" { + // no sum available, need to download + return false + } + + dir := filepath.Join(vendorDir, d.Name) + sum := hashDir(dir) + return d.Sum == sum +} + +func hashDir(dir string) string { + hasher := sha256.New() + + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(hasher, f); err != nil { + return err + } + + return nil + }) + + return base64.StdEncoding.EncodeToString(hasher.Sum(nil)) } diff --git a/spec/spec.go b/spec/spec.go index 0eafee9..c5a03cb 100644 --- a/spec/spec.go +++ b/spec/spec.go @@ -22,6 +22,7 @@ type Dependency struct { Name string `json:"name"` Source Source `json:"source"` Version string `json:"version"` + Sum string `json:"sum,omitempty"` DepSource string `json:"-"` }