technofab/jsonnet-bundler
1
0
Fork 0
mirror of https://github.com/TECHNOFAB11/jsonnet-bundler.git synced 2026-02-02 09:25:08 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: absolute imports (#63)

Browse source 
* feat: go-like import style

jb now creates a directory structure inside of vendor/ that is similar to how go
does (github.com/grafana/jsonnet-libs). This is reflected in the final import
paths, which means they will be go-like

* refactor(spec/deps): named regexs

* feat: make goImportStyle configurable

Defaults to off, can be enabled in `jsonnetfile.json`

* fix: integration test

* doc: license headers

* fix(deps): remove GO_IMPORT_STYLE

not an option anymore, will always do so and symlink

* feat: symlink to legacy location

* feat: allow to disable legacy links

* fix(test): legacyImports in integration tests

* fix(spec): test

* fix: respect legacyName aliases

It was possible to alias packages by changing `name` previously.

While names are now absolute (and computed), legacy links should still respect
old aliases to avoid breaking code.

* fix(test): integration

* fix(init): keep legacyImports enabled for now

* feat: rewrite imports

adds a command to automatically rewrite imports from legacy to absolute style

* fix(tool): rewrite confused by prefixing packages

When a package was a prefix of another one, it broke.
Fixed that by using a proper regular expression. Added a test to make sure it
works as expected

* Update cmd/jb/init.go

* fix: exclude local packages from legacy linking

They actually still use the old style, which is fine. LegacyLinking
messed them up, but from now on it just ignores symlinks that match a localPackage.
This commit is contained in:
Tom 2020-01-24 08:02:34 +01:00 • committed by GitHub
parent 184841238b
commit 7b8a7836a4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 1129 additions and 361 deletions
Download patch file Download diff file Expand all files Collapse all files

23
pkg/git.go
View file

@ -33,14 +33,14 @@ import (
"github.com/fatih/color"
"github.com/pkg/errors"
"github.com/jsonnet-bundler/jsonnet-bundler/spec"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
type GitPackage struct {
Source *spec.GitSource
Source *deps.Git
}
func NewGitPackage(source *spec.GitSource) Interface {
func NewGitPackage(source *deps.Git) Interface {
return &GitPackage{
Source: source,
}
@ -169,7 +169,7 @@ func remoteResolveRef(ctx context.Context, remote string, ref string) (string, e
func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (string, error) {
destPath := path.Join(dir, name)
tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), fmt.Sprintf("jsonnetpkg-%s-%s", name, version))
tmpDir, err := ioutil.TempDir(filepath.Join(dir, ".tmp"), fmt.Sprintf("jsonnetpkg-%s-%s", strings.Replace(name, "/", "-", -1), version))
if err != nil {
return "", errors.Wrap(err, "failed to create tmp dir")
}
@ -177,11 +177,11 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st
// Optimization for GitHub sources: download a tarball archive of the requested
// version instead of cloning the entire repository.
isGitHubRemote, err := regexp.MatchString(`^(https|ssh)://github\.com/.+$`, p.Source.Remote)
isGitHubRemote, err := regexp.MatchString(`^(https|ssh)://github\.com/.+$`, p.Source.Remote())
if isGitHubRemote {
// Let git ls-remote decide if "version" is a ref or a commit SHA in the unlikely
// but possible event that a ref is comprised of 40 or more hex characters
commitSha, err := remoteResolveRef(ctx, p.Source.Remote, version)
commitSha, err := remoteResolveRef(ctx, p.Source.Remote(), version)
// If the ref resolution failed and "version" looks like a SHA,
// assume it is one and proceed.
@ -190,7 +190,7 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st
commitSha = version
}
archiveUrl := fmt.Sprintf("%s/archive/%s.tar.gz", p.Source.Remote, commitSha)
archiveUrl := fmt.Sprintf("%s/archive/%s.tar.gz", p.Source.Remote(), commitSha)
archiveFilepath := fmt.Sprintf("%s.tar.gz", tmpDir)
defer os.Remove(archiveFilepath)
@ -205,7 +205,12 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st
// Move the extracted directory to its final destination
if err == nil {
err = os.Rename(path.Join(tmpDir, p.Source.Subdir), destPath)
if err := os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil {
panic(err)
}
if err := os.Rename(path.Join(tmpDir, p.Source.Subdir), destPath); err != nil {
panic(err)
}
}
}
}
@ -230,7 +235,7 @@ func (p *GitPackage) Install(ctx context.Context, name, dir, version string) (st
return "", err
}
cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", p.Source.Remote)
cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", p.Source.Remote())
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr

3
pkg/jsonnetfile/jsonnetfile.go
View file

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
"github.com/jsonnet-bundler/jsonnet-bundler/spec"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
const (
@ -52,7 +53,7 @@ func Unmarshal(bytes []byte) (spec.JsonnetFile, error) {
return m, errors.Wrap(err, "failed to unmarshal file")
}
if m.Dependencies == nil {
m.Dependencies = make(map[string]spec.Dependency)
m.Dependencies = make(map[string]deps.Dependency)
}
return m, nil

93
pkg/jsonnetfile/jsonnetfile_test.go
View file

@ -24,17 +24,17 @@ import (
"github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile"
"github.com/jsonnet-bundler/jsonnet-bundler/spec"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
const notExist = "/this/does/not/exist"
func TestLoad(t *testing.T) {
empty := spec.New()
jsonnetfileContent := `{
jsonnetfileContent := `
{
"legacyImports": false,
"dependencies": [
{
"name": "foobar",
"source": {
"git": {
"remote": "https://github.com/foobar/foobar",
@ -46,62 +46,61 @@ func TestLoad(t *testing.T) {
]
}
`
jsonnetFileExpected := spec.JsonnetFile{
Dependencies: map[string]spec.Dependency{
"foobar": {
Name: "foobar",
Source: spec.Source{
GitSource: &spec.GitSource{
Remote: "https://github.com/foobar/foobar",
LegacyImports: false,
Dependencies: map[string]deps.Dependency{
"github.com/foobar/foobar": {
Source: deps.Source{
GitSource: &deps.Git{
Scheme: deps.GitSchemeHTTPS,
Host: "github.com",
User: "foobar",
Repo: "foobar",
Subdir: "",
},
},
Version: "master",
DepSource: "",
Version: "master",
}},
}
{
jf, err := jsonnetfile.Load(notExist)
assert.Equal(t, empty, jf)
assert.Error(t, err)
tempDir, err := ioutil.TempDir("", "jb-load-jsonnetfile")
if err != nil {
t.Fatal(err)
}
{
tempDir, err := ioutil.TempDir("", "jb-load-jsonnetfile")
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll(tempDir)
assert.Nil(t, err)
}()
defer os.RemoveAll(tempDir)
tempFile := filepath.Join(tempDir, jsonnetfile.File)
err = ioutil.WriteFile(tempFile, []byte(`{}`), os.ModePerm)
assert.Nil(t, err)
tempFile := filepath.Join(tempDir, jsonnetfile.File)
err = ioutil.WriteFile(tempFile, []byte(jsonnetfileContent), os.ModePerm)
assert.Nil(t, err)
jf, err := jsonnetfile.Load(tempFile)
assert.Nil(t, err)
assert.Equal(t, empty, jf)
jf, err := jsonnetfile.Load(tempFile)
assert.Nil(t, err)
assert.Equal(t, jsonnetFileExpected, jf)
}
func TestLoadEmpty(t *testing.T) {
tempDir, err := ioutil.TempDir("", "jb-load-empty")
if err != nil {
t.Fatal(err)
}
{
tempDir, err := ioutil.TempDir("", "jb-load-jsonnetfile")
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll(tempDir)
assert.Nil(t, err)
}()
defer os.RemoveAll(tempDir)
tempFile := filepath.Join(tempDir, jsonnetfile.File)
err = ioutil.WriteFile(tempFile, []byte(jsonnetfileContent), os.ModePerm)
assert.Nil(t, err)
// write empty json file
tempFile := filepath.Join(tempDir, jsonnetfile.File)
err = ioutil.WriteFile(tempFile, []byte(`{}`), os.ModePerm)
assert.Nil(t, err)
jf, err := jsonnetfile.Load(tempFile)
assert.Nil(t, err)
assert.Equal(t, jsonnetFileExpected, jf)
}
// expect it to be loaded properly
got, err := jsonnetfile.Load(tempFile)
assert.Nil(t, err)
assert.Equal(t, spec.New(), got)
}
func TestLoadNotExist(t *testing.T) {
jf, err := jsonnetfile.Load(notExist)
assert.Equal(t, spec.New(), jf)
assert.Error(t, err)
}
func TestFileExists(t *testing.T) {

7
pkg/local.go
View file

@ -19,15 +19,16 @@ import (
"os"
"path/filepath"
"github.com/jsonnet-bundler/jsonnet-bundler/spec"
"github.com/pkg/errors"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
type LocalPackage struct {
Source *spec.LocalSource
Source *deps.Local
}
func NewLocalPackage(source *spec.LocalSource) Interface {
func NewLocalPackage(source *deps.Local) Interface {
return &LocalPackage{
Source: source,
}

209
pkg/packages.go
View file

@ -22,19 +22,21 @@ import (
"io"
"os"
"path/filepath"
"strings"
"github.com/fatih/color"
"github.com/pkg/errors"
"github.com/jsonnet-bundler/jsonnet-bundler/pkg/jsonnetfile"
"github.com/jsonnet-bundler/jsonnet-bundler/spec"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
var (
VersionMismatch = errors.New("multiple colliding versions specified")
)
// Ensure receives all direct packages as, the directory to vendor in and all known locks.
// Ensure receives all direct packages, the directory to vendor into and all known locks.
// It then makes sure all direct and nested dependencies are present in vendor at the correct version:
//
// If the package is locked and the files in vendor match the sha256 checksum,
@ -46,57 +48,183 @@ var (
// desired version in case by `jb install`ing it.
//
// Finally, all unknown files and directories are removed from vendor/
func Ensure(direct spec.JsonnetFile, vendorDir string, locks map[string]spec.Dependency) (map[string]spec.Dependency, error) {
// The full list of locked depedencies is returned
func Ensure(direct spec.JsonnetFile, vendorDir string, oldLocks map[string]deps.Dependency) (map[string]deps.Dependency, error) {
// ensure all required files are in vendor
deps, err := ensure(direct.Dependencies, vendorDir, locks)
// This is the actual installation
locks, err := ensure(direct.Dependencies, vendorDir, oldLocks)
if err != nil {
return nil, err
}
// cleanup unknown dirs from vendor/
f, err := os.Open(vendorDir)
if err != nil {
return nil, err
}
names, err := f.Readdirnames(0)
if err != nil {
return nil, err
}
for _, name := range names {
if _, ok := deps[name]; !ok {
dir := filepath.Join(vendorDir, name)
// remove unchanged legacyNames
CleanLegacyName(locks)
// find unknown dirs in vendor/
names := []string{}
err = filepath.Walk(vendorDir, func(path string, i os.FileInfo, err error) error {
if path == vendorDir {
return nil
}
if !i.IsDir() {
return nil
}
names = append(names, path)
return nil
})
// remove them
for _, dir := range names {
name := strings.TrimPrefix(dir, "vendor/")
if !known(locks, name) {
if err := os.RemoveAll(dir); err != nil {
return nil, err
}
if name != ".tmp" {
if !strings.HasPrefix(name, ".tmp") {
color.Magenta("CLEAN %s", dir)
}
}
}
// remove all symlinks, optionally adding known ones back later if wished
if err := cleanLegacySymlinks(vendorDir, locks); err != nil {
return nil, err
}
if !direct.LegacyImports {
return locks, nil
}
if err := linkLegacy(vendorDir, locks); err != nil {
return nil, err
}
// return the final lockfile contents
return deps, nil
return locks, nil
}
func ensure(direct map[string]spec.Dependency, vendorDir string, locks map[string]spec.Dependency) (map[string]spec.Dependency, error) {
deps := make(map[string]spec.Dependency)
func CleanLegacyName(list map[string]deps.Dependency) {
for k, d := range list {
// unset if not changed by user
if d.LegacyNameCompat == d.Source.LegacyName() {
dep := list[k]
dep.LegacyNameCompat = ""
list[k] = dep
}
}
}
func cleanLegacySymlinks(vendorDir string, locks map[string]deps.Dependency) error {
// local packages need to be ignored
locals := map[string]bool{}
for _, d := range locks {
if d.Source.LocalSource == nil {
continue
}
locals[filepath.Join(vendorDir, d.Name())] = true
}
// remove all symlinks first
return filepath.Walk(vendorDir, func(path string, i os.FileInfo, err error) error {
if locals[path] {
return nil
}
if i.Mode()&os.ModeSymlink != 0 {
if err := os.Remove(path); err != nil {
return err
}
}
return nil
})
}
func linkLegacy(vendorDir string, locks map[string]deps.Dependency) error {
// create only the ones we want
for _, d := range locks {
// localSource still uses the relative style
if d.Source.LocalSource != nil {
continue
}
legacyName := filepath.Join("vendor", d.LegacyName())
pkgName := d.Name()
taken, err := checkLegacyNameTaken(legacyName, pkgName)
if err != nil {
fmt.Println(err)
continue
}
if taken {
continue
}
// create the symlink
if err := os.Symlink(
filepath.Join(pkgName),
filepath.Join(legacyName),
); err != nil {
return err
}
}
return nil
}
func checkLegacyNameTaken(legacyName string, pkgName string) (bool, error) {
fi, err := os.Lstat(legacyName)
if err != nil {
// does not exist: not taken
if os.IsNotExist(err) {
return false, nil
}
// a real error
return false, err
}
// is it a symlink?
if fi.Mode()&os.ModeSymlink != 0 {
s, err := os.Readlink(legacyName)
if err != nil {
return false, err
}
color.Yellow("WARN: cannot link '%s' to '%s', because package '%s' already uses that name. The absolute import still works\n", pkgName, legacyName, s)
return true, nil
}
// sth else
color.Yellow("WARN: cannot link '%s' to '%s', because the file/directory already exists. The absolute import still works.\n", pkgName, legacyName)
return true, nil
}
func known(deps map[string]deps.Dependency, p string) bool {
for _, d := range deps {
k := d.Name()
if strings.HasPrefix(p, k) || strings.HasPrefix(k, p) {
return true
}
}
return false
}
func ensure(direct map[string]deps.Dependency, vendorDir string, locks map[string]deps.Dependency) (map[string]deps.Dependency, error) {
deps := make(map[string]deps.Dependency)
for _, d := range direct {
l, present := locks[d.Name]
l, present := locks[d.Name()]
// already locked and the integrity is intact
if present {
d.Version = locks[d.Name].Version
d.Version = locks[d.Name()].Version
if check(l, vendorDir) {
deps[d.Name] = l
deps[d.Name()] = l
continue
}
}
expectedSum := locks[d.Name].Sum
expectedSum := locks[d.Name()].Sum
// either not present or not intact: download again
dir := filepath.Join(vendorDir, d.Name)
dir := filepath.Join(vendorDir, d.Name())
os.RemoveAll(dir)
locked, err := download(d, vendorDir)
@ -104,15 +232,15 @@ func ensure(direct map[string]spec.Dependency, vendorDir string, locks map[strin
return nil, errors.Wrap(err, "downloading")
}
if expectedSum != "" && locked.Sum != expectedSum {
return nil, fmt.Errorf("checksum mismatch for %s. Expected %s but got %s", d.Name, expectedSum, locked.Sum)
return nil, fmt.Errorf("checksum mismatch for %s. Expected %s but got %s", d.Name(), expectedSum, locked.Sum)
}
deps[d.Name] = *locked
deps[d.Name()] = *locked
// we settled on a new version, add it to the locks for recursion
locks[d.Name] = *locked
locks[d.Name()] = *locked
}
for _, d := range deps {
f, err := jsonnetfile.Load(filepath.Join(vendorDir, d.Name, jsonnetfile.File))
f, err := jsonnetfile.Load(filepath.Join(vendorDir, d.Name(), jsonnetfile.File))
if err != nil {
if os.IsNotExist(err) {
continue
@ -126,8 +254,8 @@ func ensure(direct map[string]spec.Dependency, vendorDir string, locks map[strin
}
for _, d := range nested {
if _, ok := deps[d.Name]; !ok {
deps[d.Name] = d
if _, ok := deps[d.Name()]; !ok {
deps[d.Name()] = d
}
}
}
@ -137,7 +265,7 @@ func ensure(direct map[string]spec.Dependency, vendorDir string, locks map[strin
// download retrieves a package from a remote upstream. The checksum of the
// files is generated afterwards.
func download(d spec.Dependency, vendorDir string) (*spec.Dependency, error) {
func download(d deps.Dependency, vendorDir string) (*deps.Dependency, error) {
var p Interface
switch {
case d.Source.GitSource != nil:
@ -150,32 +278,29 @@ func download(d spec.Dependency, vendorDir string) (*spec.Dependency, error) {
return nil, errors.New("either git or local source is required")
}
version, err := p.Install(context.TODO(), d.Name, vendorDir, d.Version)
version, err := p.Install(context.TODO(), d.Name(), vendorDir, d.Version)
if err != nil {
return nil, err
}
var sum string
if d.Source.LocalSource == nil {
sum = hashDir(filepath.Join(vendorDir, d.Name))
sum = hashDir(filepath.Join(vendorDir, d.Name()))
}
return &spec.Dependency{
Name: d.Name,
Source: d.Source,
Version: version,
Sum: sum,
}, nil
d.Version = version
d.Sum = sum
return &d, nil
}
// check returns whether the files present at the vendor/ folder match the
// sha256 sum of the package. local-directory dependencies are not checked as
// their purpose is to change during development where integrity checking would
// be a hindrance.
func check(d spec.Dependency, vendorDir string) bool {
func check(d deps.Dependency, vendorDir string) bool {
// assume a local dependency is intact as long as it exists
if d.Source.LocalSource != nil {
x, err := jsonnetfile.Exists(filepath.Join(vendorDir, d.Name))
x, err := jsonnetfile.Exists(filepath.Join(vendorDir, d.Name()))
if err != nil {
return false
}
@ -187,7 +312,7 @@ func check(d spec.Dependency, vendorDir string) bool {
return false
}
dir := filepath.Join(vendorDir, d.Name)
dir := filepath.Join(vendorDir, d.Name())
sum := hashDir(dir)
return d.Sum == sum
}

80
pkg/packages_test.go
View file

@ -13,3 +13,83 @@
// limitations under the License.
package pkg
import (
"testing"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
)
func TestKnown(t *testing.T) {
deps := map[string]deps.Dependency{
"ksonnet-lib": deps.Dependency{
Source: deps.Source{GitSource: &deps.Git{
Scheme: deps.GitSchemeHTTPS,
Host: "github.com",
User: "ksonnet",
Repo: "ksonnet-lib",
Subdir: "/ksonnet.beta.4",
}},
},
}
paths := []string{
"github.com",
"github.com/ksonnet",
"github.com/ksonnet/ksonnet-lib",
"github.com/ksonnet/ksonnet-lib/ksonnet.beta.4",
"github.com/ksonnet/ksonnet-lib/ksonnet.beta.4/k.libsonnet",
"github.com/ksonnet-util", // don't know that one
"ksonnet.beta.4", // the symlink
}
want := []string{
"github.com",
"github.com/ksonnet",
"github.com/ksonnet/ksonnet",
"github.com/ksonnet/ksonnet-lib",
"github.com/ksonnet/ksonnet-lib/ksonnet.beta.4",
"github.com/ksonnet/ksonnet-lib/ksonnet.beta.4/k.libsonnet",
}
w := make(map[string]bool)
for _, k := range want {
w[k] = true
}
for _, p := range paths {
if known(deps, p) != w[p] {
t.Fatalf("expected %s to be %v", p, w[p])
}
}
}
func TestCleanLegacyName(t *testing.T) {
deps := func(name string) map[string]deps.Dependency {
return map[string]deps.Dependency{
"ksonnet-lib": deps.Dependency{
LegacyNameCompat: name,
Source: deps.Source{GitSource: &deps.Git{
Scheme: deps.GitSchemeHTTPS,
Host: "github.com",
User: "ksonnet",
Repo: "ksonnet-lib",
Subdir: "/ksonnet.beta.4",
}},
},
}
}
cases := map[string]bool{
"ksonnet": false,
"ksonnet.beta.4": true,
}
for name, want := range cases {
list := deps(name)
CleanLegacyName(list)
if (list["ksonnet-lib"].LegacyNameCompat == "") != want {
t.Fatalf("expected `%s` to be removed: %v", name, want)
}
}
}