feat: absolute imports (#63)

* feat: go-like import style

jb now creates a directory structure inside of vendor/ that is similar to how go
does (github.com/grafana/jsonnet-libs). This is reflected in the final import
paths, which means they will be go-like

* refactor(spec/deps): named regexs

* feat: make goImportStyle configurable

Defaults to off, can be enabled in `jsonnetfile.json`

* fix: integration test

* doc: license headers

* fix(deps): remove GO_IMPORT_STYLE

not an option anymore, will always do so and symlink

* feat: symlink to legacy location

* feat: allow to disable legacy links

* fix(test): legacyImports in integration tests

* fix(spec): test

* fix: respect legacyName aliases

It was possible to alias packages by changing `name` previously.

While names are now absolute (and computed), legacy links should still respect
old aliases to avoid breaking code.

* fix(test): integration

* fix(init): keep legacyImports enabled for now

* feat: rewrite imports

adds a command to automatically rewrite imports from legacy to absolute style

* fix(tool): rewrite confused by prefixing packages

When a package was a prefix of another one, it broke.
Fixed that by using a proper regular expression. Added a test to make sure it
works as expected

* Update cmd/jb/init.go

* fix: exclude local packages from legacy linking

They actually still use the old style, which is fine. LegacyLinking
messed them up, but from now on it just ignores symlinks that match a localPackage.

spec/deps/dependencies.go

@@ -0,0 +1,106 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package deps
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type Dependency struct {
+	Source  Source `json:"source"`
+	Version string `json:"version"`
+	Sum     string `json:"sum,omitempty"`
+
+	// older schema used to have `name`. We still need that data for
+	// `LegacyName`
+	LegacyNameCompat string `json:"name,omitempty"`
+}
+
+func Parse(dir, uri string) *Dependency {
+	if uri == "" {
+		return nil
+	}
+
+	if d := parseGit(uri); d != nil {
+		return d
+	}
+
+	return parseLocal(dir, uri)
+}
+
+func (d Dependency) Name() string {
+	return d.Source.Name()
+}
+
+func (d Dependency) LegacyName() string {
+	if d.LegacyNameCompat != "" {
+		return d.LegacyNameCompat
+	}
+	return d.Source.LegacyName()
+}
+
+type Source struct {
+	GitSource   *Git   `json:"git,omitempty"`
+	LocalSource *Local `json:"local,omitempty"`
+}
+
+func (s Source) Name() string {
+	switch {
+	case s.GitSource != nil:
+		return s.GitSource.Name()
+	case s.LocalSource != nil:
+		return s.LegacyName()
+	default:
+		return ""
+	}
+}
+
+func (s Source) LegacyName() string {
+	switch {
+	case s.GitSource != nil:
+		return s.GitSource.LegacyName()
+	case s.LocalSource != nil:
+		return filepath.Base(s.LocalSource.Directory)
+	default:
+		return ""
+	}
+}
+
+type Local struct {
+	Directory string `json:"directory"`
+}
+
+func parseLocal(dir, p string) *Dependency {
+	clean := filepath.Clean(p)
+	abs := filepath.Join(dir, clean)
+
+	info, err := os.Stat(abs)
+	if err != nil {
+		return nil
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	return &Dependency{
+		Source: Source{
+			LocalSource: &Local{
+				Directory: clean,
+			},
+		},
+		Version: "",
+	}
+}

spec/deps/dependencies_test.go

@@ -0,0 +1,70 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package deps
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseDependency(t *testing.T) {
+	const testFolder = "test/jsonnet/foobar"
+	err := os.MkdirAll(testFolder, os.ModePerm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll("test")
+
+	tests := []struct {
+		name string
+		path string
+		want *Dependency
+	}{
+		{
+			name: "Empty",
+			path: "",
+			want: nil,
+		},
+		{
+			name: "Invalid",
+			path: "github.com/foo",
+			want: nil,
+		},
+		{
+			name: "local",
+			path: testFolder,
+			want: &Dependency{
+				Source: Source{
+					LocalSource: &Local{
+						Directory: "test/jsonnet/foobar",
+					},
+				},
+				Version: "",
+			},
+		},
+	}
+	for _, tt := range tests {
+		_ = t.Run(tt.name, func(t *testing.T) {
+			dependency := Parse("", tt.path)
+
+			if tt.path == "" {
+				assert.Nil(t, dependency)
+			} else {
+				assert.Equal(t, tt.want, dependency)
+			}
+		})
+	}
+}

spec/deps/git.go

@@ -0,0 +1,203 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package deps
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+const (
+	GitSchemeSSH   = "ssh://git@"
+	GitSchemeHTTPS = "https://"
+)
+
+// Git holds all required information for cloning a package from git
+type Git struct {
+	// Scheme (Protocol) used (https, git+ssh)
+	Scheme string
+
+	// Hostname the repo is located at
+	Host string
+	// User (github.com/<user>)
+	User string
+	// Repo (github.com/<user>/<repo>)
+	Repo string
+	// Subdir (github.com/<user>/<repo>/<subdir>)
+	Subdir string
+}
+
+// json representation of Git (for compatiblity with old format)
+type jsonGit struct {
+	Remote string `json:"remote"`
+	Subdir string `json:"subdir"`
+}
+
+// MarshalJSON takes care of translating between Git and jsonGit
+func (gs *Git) MarshalJSON() ([]byte, error) {
+	j := jsonGit{
+		Remote: gs.Remote(),
+		Subdir: strings.TrimPrefix(gs.Subdir, "/"),
+	}
+	return json.Marshal(j)
+}
+
+// UnmarshalJSON takes care of translating between Git and jsonGit
+func (gs *Git) UnmarshalJSON(data []byte) error {
+	var j jsonGit
+	if err := json.Unmarshal(data, &j); err != nil {
+		return err
+	}
+
+	if j.Subdir != "" {
+		gs.Subdir = "/" + strings.TrimPrefix(j.Subdir, "/")
+	}
+
+	tmp := parseGit(j.Remote)
+	gs.Host = tmp.Source.GitSource.Host
+	gs.User = tmp.Source.GitSource.User
+	gs.Repo = tmp.Source.GitSource.Repo
+	gs.Scheme = tmp.Source.GitSource.Scheme
+	return nil
+}
+
+// Name returns the repository in a go-like format (github.com/user/repo/subdir)
+func (gs *Git) Name() string {
+	return fmt.Sprintf("%s/%s/%s%s", gs.Host, gs.User, gs.Repo, gs.Subdir)
+}
+
+// LegacyName returns the last element of the packages path
+// example: github.com/ksonnet/ksonnet-lib/ksonnet.beta.4 becomes ksonnet.beta.4
+func (gs *Git) LegacyName() string {
+	return filepath.Base(gs.Repo + gs.Subdir)
+}
+
+var gitProtoFmts = map[string]string{
+	GitSchemeSSH:   GitSchemeSSH + "%s:%s/%s.git",
+	GitSchemeHTTPS: GitSchemeHTTPS + "%s/%s/%s",
+}
+
+// Remote returns a remote string that can be passed to git
+func (gs *Git) Remote() string {
+	return fmt.Sprintf(gitProtoFmts[gs.Scheme],
+		gs.Host, gs.User, gs.Repo,
+	)
+}
+
+// regular expressions for matching package uris
+const (
+	gitSSHExp     = `git\+ssh://git@(?P<host>[^:]+):(?P<user>[^/]+)/(?P<repo>[^/]+).git`
+	githubSlugExp = `github.com/(?P<user>[-_a-zA-Z0-9]+)/(?P<repo>[-_a-zA-Z0-9]+)`
+)
+
+var (
+	gitSSHRegex                   = regexp.MustCompile(gitSSHExp)
+	gitSSHWithVersionRegex        = regexp.MustCompile(gitSSHExp + `@(?P<version>.*)`)
+	gitSSHWithPathRegex           = regexp.MustCompile(gitSSHExp + `/(?P<subdir>.*)`)
+	gitSSHWithPathAndVersionRegex = regexp.MustCompile(gitSSHExp + `/(?P<subdir>.*)@(?P<version>.*)`)
+
+	githubSlugRegex                   = regexp.MustCompile(githubSlugExp)
+	githubSlugWithVersionRegex        = regexp.MustCompile(githubSlugExp + `@(?P<version>.*)`)
+	githubSlugWithPathRegex           = regexp.MustCompile(githubSlugExp + `/(?P<subdir>.*)`)
+	githubSlugWithPathAndVersionRegex = regexp.MustCompile(githubSlugExp + `/(?P<subdir>.*)@(?P<version>.*)`)
+)
+
+func parseGit(uri string) *Dependency {
+	var d = Dependency{
+		Version: "master",
+		Source:  Source{},
+	}
+	var gs *Git
+	var version string
+
+	switch {
+	case githubSlugRegex.MatchString(uri):
+		gs, version = parseGitHub(uri)
+	case gitSSHRegex.MatchString(uri):
+		gs, version = parseGitSSH(uri)
+	default:
+		return nil
+	}
+
+	if gs.Subdir != "" {
+		gs.Subdir = "/" + gs.Subdir
+	}
+
+	d.Source.GitSource = gs
+	if version != "" {
+		d.Version = version
+	}
+	return &d
+}
+
+func parseGitSSH(p string) (gs *Git, version string) {
+	gs, version = match(p, []*regexp.Regexp{
+		gitSSHWithPathAndVersionRegex,
+		gitSSHWithPathRegex,
+		gitSSHWithVersionRegex,
+		gitSSHRegex,
+	})
+
+	gs.Scheme = GitSchemeSSH
+	return gs, version
+}
+
+func parseGitHub(p string) (gs *Git, version string) {
+	gs, version = match(p, []*regexp.Regexp{
+		githubSlugWithPathAndVersionRegex,
+		githubSlugWithPathRegex,
+		githubSlugWithVersionRegex,
+		githubSlugRegex,
+	})
+
+	gs.Scheme = GitSchemeHTTPS
+	gs.Host = "github.com"
+	return gs, version
+}
+
+func match(p string, exps []*regexp.Regexp) (gs *Git, version string) {
+	gs = &Git{}
+	for _, e := range exps {
+		if !e.MatchString(p) {
+			continue
+		}
+
+		matches := reSubMatchMap(e, p)
+		gs.Host = matches["host"]
+		gs.User = matches["user"]
+		gs.Repo = matches["repo"]
+
+		if sd, ok := matches["subdir"]; ok {
+			gs.Subdir = sd
+		}
+
+		return gs, matches["version"]
+	}
+	return gs, ""
+}
+
+func reSubMatchMap(r *regexp.Regexp, str string) map[string]string {
+	match := r.FindStringSubmatch(str)
+	subMatchMap := make(map[string]string)
+	for i, name := range r.SubexpNames() {
+		if i != 0 {
+			subMatchMap[name] = match[i]
+		}
+	}
+
+	return subMatchMap
+}

spec/deps/git_test.go

@@ -0,0 +1,68 @@
+// Copyright 2018 jsonnet-bundler authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package deps
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseGit(t *testing.T) {
+	tests := []struct {
+		name       string
+		uri        string
+		want       *Dependency
+		wantRemote string
+	}{
+		{
+			name: "GitHub",
+			uri:  "github.com/ksonnet/ksonnet-lib/ksonnet.beta.3",
+			want: &Dependency{
+				Version: "master",
+				Source: Source{GitSource: &Git{
+					Scheme: GitSchemeHTTPS,
+					Host:   "github.com",
+					User:   "ksonnet",
+					Repo:   "ksonnet-lib",
+					Subdir: "/ksonnet.beta.3",
+				}},
+			},
+			wantRemote: "https://github.com/ksonnet/ksonnet-lib",
+		},
+		{
+			name: "SSH",
+			uri:  "git+ssh://git@my.host:user/repo.git/foobar@v1",
+			want: &Dependency{
+				Version: "v1",
+				Source: Source{GitSource: &Git{
+					Scheme: GitSchemeSSH,
+					Host:   "my.host",
+					User:   "user",
+					Repo:   "repo",
+					Subdir: "/foobar",
+				}},
+			},
+			wantRemote: "ssh://git@my.host:user/repo.git",
+		},
+	}
+
+	for _, c := range tests {
+		t.Run(c.name, func(t *testing.T) {
+			got := Parse("", c.uri)
+			assert.Equal(t, c.want, got)
+			assert.Equal(t, c.wantRemote, got.Source.GitSource.Remote())
+		})
+	}
+}

spec/spec.go

@@ -17,73 +17,67 @@ package spec
 import (
 	"encoding/json"
 	"sort"
+
+	"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
 )
 
 // JsonnetFile is the structure of a `.json` file describing a set of jsonnet
 // dependencies. It is used for both, the jsonnetFile and the lockFile.
 type JsonnetFile struct {
-	Dependencies map[string]Dependency
+	// List of dependencies
+	Dependencies map[string]deps.Dependency
+
+	// Symlink files to old location
+	LegacyImports bool
 }
 
 // New returns a new JsonnetFile with the dependencies map initialized
 func New() JsonnetFile {
 	return JsonnetFile{
-		Dependencies: make(map[string]Dependency),
+		Dependencies:  make(map[string]deps.Dependency),
+		LegacyImports: true,
 	}
 }
 
 // jsonFile is the json representation of a JsonnetFile, which is different for
 // compatibility reasons.
 type jsonFile struct {
-	Dependencies []Dependency `json:"dependencies"`
+	Dependencies  []deps.Dependency `json:"dependencies"`
+	LegacyImports bool              `json:"legacyImports"`
 }
 
 // UnmarshalJSON unmarshals a `jsonFile`'s json into a JsonnetFile
 func (jf *JsonnetFile) UnmarshalJSON(data []byte) error {
 	var s jsonFile
+	s.LegacyImports = jf.LegacyImports // adpot default
+
 	if err := json.Unmarshal(data, &s); err != nil {
 		return err
 	}
 
-	jf.Dependencies = make(map[string]Dependency)
+	jf.Dependencies = make(map[string]deps.Dependency)
 	for _, d := range s.Dependencies {
-		jf.Dependencies[d.Name] = d
+		jf.Dependencies[d.Name()] = d
 	}
+	jf.LegacyImports = s.LegacyImports
 	return nil
 }
 
 // MarshalJSON serializes a JsonnetFile into json of the format of a `jsonFile`
 func (jf JsonnetFile) MarshalJSON() ([]byte, error) {
 	var s jsonFile
+	s.LegacyImports = jf.LegacyImports
 	for _, d := range jf.Dependencies {
 		s.Dependencies = append(s.Dependencies, d)
 	}
 
 	sort.SliceStable(s.Dependencies, func(i int, j int) bool {
-		return s.Dependencies[i].Name < s.Dependencies[j].Name
+		return s.Dependencies[i].Name() < s.Dependencies[j].Name()
 	})
 
+	if s.Dependencies == nil {
+		s.Dependencies = make([]deps.Dependency, 0, 0)
+	}
+
 	return json.Marshal(s)
 }
-
-type Dependency struct {
-	Name      string `json:"name"`
-	Source    Source `json:"source"`
-	Version   string `json:"version"`
-	Sum       string `json:"sum,omitempty"`
-	DepSource string `json:"-"`
-}
-
-type Source struct {
-	GitSource   *GitSource   `json:"git,omitempty"`
-	LocalSource *LocalSource `json:"local,omitempty"`
-}
-
-type GitSource struct {
-	Remote string `json:"remote"`
-	Subdir string `json:"subdir"`
-}
-
-type LocalSource struct {
-	Directory string `json:"directory"`
-}

spec/spec_test.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/jsonnet-bundler/jsonnet-bundler/spec/deps"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,7 +26,6 @@ import (
 const jsonJF = `{
   "dependencies": [
     {
-      "name": "grafana-builder",
       "source": {
         "git": {
           "remote": "https://github.com/grafana/jsonnet-libs",
@@ -36,7 +36,7 @@ const jsonJF = `{
       "sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE="
     },
     {
-      "name": "prometheus-mixin",
+      "name": "prometheus",
       "source": {
         "git": {
           "remote": "https://github.com/prometheus/prometheus",
@@ -46,29 +46,36 @@ const jsonJF = `{
       "version": "7c039a6b3b4b2a9d7c613ac8bd3fc16e8ca79684",
       "sum": "bVGOsq3hLOw2irNPAS91a5dZJqQlBUNWy3pVwM4+kIY="
     }
-  ]
+  ],
+  "legacyImports": false
 }`
 
 func testData() JsonnetFile {
 	return JsonnetFile{
-		Dependencies: map[string]Dependency{
-			"grafana-builder": {
-				Name: "grafana-builder",
-				Source: Source{
-					GitSource: &GitSource{
-						Remote: "https://github.com/grafana/jsonnet-libs",
-						Subdir: "grafana-builder",
+		LegacyImports: false,
+		Dependencies: map[string]deps.Dependency{
+			"github.com/grafana/jsonnet-libs/grafana-builder": {
+				Source: deps.Source{
+					GitSource: &deps.Git{
+						Scheme: deps.GitSchemeHTTPS,
+						Host:   "github.com",
+						User:   "grafana",
+						Repo:   "jsonnet-libs",
+						Subdir: "/grafana-builder",
 					},
 				},
 				Version: "54865853ebc1f901964e25a2e7a0e4d2cb6b9648",
 				Sum:     "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=",
 			},
-			"prometheus-mixin": {
-				Name: "prometheus-mixin",
-				Source: Source{
-					GitSource: &GitSource{
-						Remote: "https://github.com/prometheus/prometheus",
-						Subdir: "documentation/prometheus-mixin",
+			"github.com/prometheus/prometheus/documentation/prometheus-mixin": {
+				LegacyNameCompat: "prometheus",
+				Source: deps.Source{
+					GitSource: &deps.Git{
+						Scheme: deps.GitSchemeHTTPS,
+						Host:   "github.com",
+						User:   "prometheus",
+						Repo:   "prometheus",
+						Subdir: "/documentation/prometheus-mixin",
 					},
 				},
 				Version: "7c039a6b3b4b2a9d7c613ac8bd3fc16e8ca79684",