feat(spec): version field (#85)

Adds a `version` field to the `jsonnetfile.json`, so that `jb` can automatically recognize too old / too new schema versions, instead of panicking.
This commit is contained in:
Matthias Loibl 2020-02-28 17:41:49 +01:00 committed by GitHub
parent efe0c9e864
commit bcd89fd33d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 454 additions and 102 deletions

View file

@ -0,0 +1,106 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deps
import (
"os"
"path/filepath"
)
type Dependency struct {
Source Source `json:"source"`
Version string `json:"version"`
Sum string `json:"sum,omitempty"`
// older schema used to have `name`. We still need that data for
// `LegacyName`
LegacyNameCompat string `json:"name,omitempty"`
}
func Parse(dir, uri string) *Dependency {
if uri == "" {
return nil
}
if d := parseGit(uri); d != nil {
return d
}
return parseLocal(dir, uri)
}
func (d Dependency) Name() string {
return d.Source.Name()
}
func (d Dependency) LegacyName() string {
if d.LegacyNameCompat != "" {
return d.LegacyNameCompat
}
return d.Source.LegacyName()
}
type Source struct {
GitSource *Git `json:"git,omitempty"`
LocalSource *Local `json:"local,omitempty"`
}
func (s Source) Name() string {
switch {
case s.GitSource != nil:
return s.GitSource.Name()
case s.LocalSource != nil:
return s.LegacyName()
default:
return ""
}
}
func (s Source) LegacyName() string {
switch {
case s.GitSource != nil:
return s.GitSource.LegacyName()
case s.LocalSource != nil:
return filepath.Base(s.LocalSource.Directory)
default:
return ""
}
}
type Local struct {
Directory string `json:"directory"`
}
func parseLocal(dir, p string) *Dependency {
clean := filepath.Clean(p)
abs := filepath.Join(dir, clean)
info, err := os.Stat(abs)
if err != nil {
return nil
}
if !info.IsDir() {
return nil
}
return &Dependency{
Source: Source{
LocalSource: &Local{
Directory: clean,
},
},
Version: "",
}
}

View file

@ -0,0 +1,80 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deps
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseDependency(t *testing.T) {
const testFolder = "test/jsonnet/foobar"
err := os.MkdirAll(testFolder, os.ModePerm)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("test")
tests := []struct {
name string
path string
want *Dependency
}{
{
name: "Empty",
path: "",
want: nil,
},
{
name: "Invalid",
path: "example.com/foo",
want: nil,
},
{
name: "InvalidDomain",
path: "example.c/foo/bar",
want: nil,
},
{
name: "InvalidDomain2",
path: "examplec/foo/bar",
want: nil,
},
{
name: "local",
path: testFolder,
want: &Dependency{
Source: Source{
LocalSource: &Local{
Directory: "test/jsonnet/foobar",
},
},
Version: "",
},
},
}
for _, tt := range tests {
_ = t.Run(tt.name, func(t *testing.T) {
dependency := Parse("", tt.path)
if tt.path == "" {
assert.Nil(t, dependency)
} else {
assert.Equal(t, tt.want, dependency)
}
})
}
}

193
spec/v1/deps/git.go Normal file
View file

@ -0,0 +1,193 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deps
import (
"encoding/json"
"fmt"
"path/filepath"
"regexp"
"strings"
)
const (
GitSchemeSSH = "ssh://git@"
GitSchemeHTTPS = "https://"
)
// Git holds all required information for cloning a package from git
type Git struct {
// Scheme (Protocol) used (https, git+ssh)
Scheme string
// Hostname the repo is located at
Host string
// User (example.com/<user>)
User string
// Repo (example.com/<user>/<repo>)
Repo string
// Subdir (example.com/<user>/<repo>/<subdir>)
Subdir string
}
// json representation of Git (for compatiblity with old format)
type jsonGit struct {
Remote string `json:"remote"`
Subdir string `json:"subdir"`
}
// MarshalJSON takes care of translating between Git and jsonGit
func (gs *Git) MarshalJSON() ([]byte, error) {
j := jsonGit{
Remote: gs.Remote(),
Subdir: strings.TrimPrefix(gs.Subdir, "/"),
}
return json.Marshal(j)
}
// UnmarshalJSON takes care of translating between Git and jsonGit
func (gs *Git) UnmarshalJSON(data []byte) error {
var j jsonGit
if err := json.Unmarshal(data, &j); err != nil {
return err
}
if j.Subdir != "" {
gs.Subdir = "/" + strings.TrimPrefix(j.Subdir, "/")
}
tmp := parseGit(j.Remote)
if tmp == nil {
return fmt.Errorf("unable to parse git url `%s` ", j.Remote)
}
gs.Host = tmp.Source.GitSource.Host
gs.User = tmp.Source.GitSource.User
gs.Repo = tmp.Source.GitSource.Repo
gs.Scheme = tmp.Source.GitSource.Scheme
return nil
}
// Name returns the repository in a go-like format (example.com/user/repo/subdir)
func (gs *Git) Name() string {
return fmt.Sprintf("%s/%s/%s%s", gs.Host, gs.User, gs.Repo, gs.Subdir)
}
// LegacyName returns the last element of the packages path
// example: github.com/ksonnet/ksonnet-lib/ksonnet.beta.4 becomes ksonnet.beta.4
func (gs *Git) LegacyName() string {
return filepath.Base(gs.Repo + gs.Subdir)
}
var gitProtoFmts = map[string]string{
GitSchemeSSH: GitSchemeSSH + "%s/%s/%s.git",
GitSchemeHTTPS: GitSchemeHTTPS + "%s/%s/%s",
}
// Remote returns a remote string that can be passed to git
func (gs *Git) Remote() string {
return fmt.Sprintf(gitProtoFmts[gs.Scheme],
gs.Host, gs.User, gs.Repo,
)
}
// regular expressions for matching package uris
const (
gitSSHExp = `ssh://git@(?P<host>.+)/(?P<user>.+)/(?P<repo>.+).git`
gitSCPExp = `^git@(?P<host>.+):(?P<user>.+)/(?P<repo>.+).git`
// The long ugly pattern for ${host} here is a generic pattern for "valid URL with zero or more subdomains and a valid TLD"
gitHTTPSExp = `(?P<host>[a-zA-Z0-9][a-zA-Z0-9-\.]{1,61}[a-zA-Z0-9]\.[a-zA-Z]{2,})/(?P<user>[-_a-zA-Z0-9]+)/(?P<repo>[-_a-zA-Z0-9]+)`
)
var (
VersionRegex = `@(?P<version>.*)`
PathRegex = `/(?P<subdir>.*)`
PathAndVersionRegex = `/(?P<subdir>.*)@(?P<version>.*)`
)
func parseGit(uri string) *Dependency {
var d = Dependency{
Version: "master",
Source: Source{},
}
var gs *Git
var version string
switch {
case reMatch(gitSSHExp, uri):
gs, version = match(uri, gitSSHExp)
gs.Scheme = GitSchemeSSH
case reMatch(gitSCPExp, uri):
gs, version = match(uri, gitSCPExp)
gs.Scheme = GitSchemeSSH
case reMatch(gitHTTPSExp, uri):
gs, version = match(uri, gitHTTPSExp)
gs.Scheme = GitSchemeHTTPS
default:
return nil
}
if gs.Subdir != "" {
gs.Subdir = "/" + gs.Subdir
}
d.Source.GitSource = gs
if version != "" {
d.Version = version
}
return &d
}
func match(p string, exp string) (gs *Git, version string) {
gs = &Git{}
exps := []*regexp.Regexp{
regexp.MustCompile(exp + PathAndVersionRegex),
regexp.MustCompile(exp + PathRegex),
regexp.MustCompile(exp + VersionRegex),
regexp.MustCompile(exp),
}
for _, e := range exps {
if !e.MatchString(p) {
continue
}
matches := reSubMatchMap(e, p)
gs.Host = matches["host"]
gs.User = matches["user"]
gs.Repo = matches["repo"]
if sd, ok := matches["subdir"]; ok {
gs.Subdir = sd
}
return gs, matches["version"]
}
return gs, ""
}
func reMatch(exp string, str string) bool {
return regexp.MustCompile(exp).MatchString(str)
}
func reSubMatchMap(r *regexp.Regexp, str string) map[string]string {
match := r.FindStringSubmatch(str)
subMatchMap := make(map[string]string)
for i, name := range r.SubexpNames() {
if i != 0 {
subMatchMap[name] = match[i]
}
}
return subMatchMap
}

186
spec/v1/deps/git_test.go Normal file
View file

@ -0,0 +1,186 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deps
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseGit(t *testing.T) {
sshWant := func(host string) *Dependency {
return &Dependency{
Version: "v1",
Source: Source{GitSource: &Git{
Scheme: GitSchemeSSH,
Host: host,
User: "user",
Repo: "repo",
Subdir: "/foobar",
}},
}
}
tests := []struct {
name string
uri string
want *Dependency
wantRemote string
}{
{
name: "github-slug",
uri: "github.com/ksonnet/ksonnet-lib/ksonnet.beta.3",
want: &Dependency{
Version: "master",
Source: Source{GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "github.com",
User: "ksonnet",
Repo: "ksonnet-lib",
Subdir: "/ksonnet.beta.3",
}},
},
wantRemote: "https://github.com/ksonnet/ksonnet-lib",
},
{
name: "ssh.ssh",
uri: "ssh://git@example.com/user/repo.git/foobar@v1",
want: sshWant("example.com"),
wantRemote: "ssh://git@example.com/user/repo.git",
},
{
name: "ssh.scp",
uri: "git@my.host:user/repo.git/foobar@v1",
want: sshWant("my.host"),
wantRemote: "ssh://git@my.host/user/repo.git", // want ssh format here
},
{
name: "ValidGitHTTPS",
uri: "https://example.com/foo/bar",
want: &Dependency{
Version: "master",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "example.com",
User: "foo",
Repo: "bar",
Subdir: "",
},
},
},
wantRemote: "https://example.com/foo/bar",
},
{
name: "ValidGitNoScheme",
uri: "example.com/foo/bar",
want: &Dependency{
Version: "master",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "example.com",
User: "foo",
Repo: "bar",
Subdir: "",
},
},
},
wantRemote: "https://example.com/foo/bar",
},
{
name: "ValidGitPath",
uri: "example.com/foo/bar/baz/bat",
want: &Dependency{
Version: "master",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "example.com",
User: "foo",
Repo: "bar",
Subdir: "/baz/bat",
},
},
},
wantRemote: "https://example.com/foo/bar",
},
{
name: "ValidGitVersion",
uri: "example.com/foo/bar@baz",
want: &Dependency{
Version: "baz",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "example.com",
User: "foo",
Repo: "bar",
Subdir: "",
},
},
},
wantRemote: "https://example.com/foo/bar",
},
{
name: "ValidGitPathVersion",
uri: "example.com/foo/bar/baz@bat",
want: &Dependency{
Version: "bat",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "example.com",
User: "foo",
Repo: "bar",
Subdir: "/baz",
},
},
},
wantRemote: "https://example.com/foo/bar",
},
{
name: "ValidGitSubdomain",
uri: "git.example.com/foo/bar",
want: &Dependency{
Version: "master",
Source: Source{
GitSource: &Git{
Scheme: GitSchemeHTTPS,
Host: "git.example.com",
User: "foo",
Repo: "bar",
Subdir: "",
},
},
},
wantRemote: "https://git.example.com/foo/bar",
},
}
for _, c := range tests {
t.Run(c.name, func(t *testing.T) {
got := Parse("", c.uri)
require.NotNilf(t, got, "parsed dependency is nil. Most likely, no regex matched the format.")
assert.Equal(t, c.want, got)
require.NotNil(t, got.Source)
require.NotNil(t, got.Source.GitSource)
assert.Equal(t, c.wantRemote, got.Source.GitSource.Remote())
})
}
}

90
spec/v1/spec.go Normal file
View file

@ -0,0 +1,90 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package spec
import (
"encoding/json"
"sort"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps"
)
const Version uint = 1
// JsonnetFile is the structure of a `.json` file describing a set of jsonnet
// dependencies. It is used for both, the jsonnetFile and the lockFile.
type JsonnetFile struct {
// List of dependencies
Dependencies map[string]deps.Dependency
// Symlink files to old location
LegacyImports bool
}
// New returns a new JsonnetFile with the dependencies map initialized
func New() JsonnetFile {
return JsonnetFile{
Dependencies: make(map[string]deps.Dependency),
LegacyImports: true,
}
}
// jsonFile is the json representation of a JsonnetFile, which is different for
// compatibility reasons.
type jsonFile struct {
Version uint `json:"version"`
Dependencies []deps.Dependency `json:"dependencies"`
LegacyImports bool `json:"legacyImports"`
}
// UnmarshalJSON unmarshals a `jsonFile`'s json into a JsonnetFile
func (jf *JsonnetFile) UnmarshalJSON(data []byte) error {
var s jsonFile
s.LegacyImports = jf.LegacyImports // adpot default
if err := json.Unmarshal(data, &s); err != nil {
return err
}
jf.Dependencies = make(map[string]deps.Dependency)
for _, d := range s.Dependencies {
jf.Dependencies[d.Name()] = d
}
jf.LegacyImports = s.LegacyImports
return nil
}
// MarshalJSON serializes a JsonnetFile into json of the format of a `jsonFile`
func (jf JsonnetFile) MarshalJSON() ([]byte, error) {
var s jsonFile
s.Version = Version
for _, d := range jf.Dependencies {
s.Dependencies = append(s.Dependencies, d)
}
sort.SliceStable(s.Dependencies, func(i int, j int) bool {
return s.Dependencies[i].Name() < s.Dependencies[j].Name()
})
if s.Dependencies == nil {
s.Dependencies = make([]deps.Dependency, 0, 0)
}
return json.Marshal(s)
}

118
spec/v1/spec_test.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2018 jsonnet-bundler authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package spec
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/jsonnet-bundler/jsonnet-bundler/spec/v1/deps"
)
const jsonJF = `{
"version": 1,
"dependencies": [
{
"source": {
"git": {
"remote": "https://github.com/grafana/jsonnet-libs",
"subdir": "grafana-builder"
}
},
"version": "54865853ebc1f901964e25a2e7a0e4d2cb6b9648",
"sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE="
},
{
"name": "prometheus",
"source": {
"git": {
"remote": "https://github.com/prometheus/prometheus",
"subdir": "documentation/prometheus-mixin"
}
},
"version": "7c039a6b3b4b2a9d7c613ac8bd3fc16e8ca79684",
"sum": "bVGOsq3hLOw2irNPAS91a5dZJqQlBUNWy3pVwM4+kIY="
}
],
"legacyImports": false
}`
func testData() JsonnetFile {
return JsonnetFile{
LegacyImports: false,
Dependencies: map[string]deps.Dependency{
"github.com/grafana/jsonnet-libs/grafana-builder": {
Source: deps.Source{
GitSource: &deps.Git{
Scheme: deps.GitSchemeHTTPS,
Host: "github.com",
User: "grafana",
Repo: "jsonnet-libs",
Subdir: "/grafana-builder",
},
},
Version: "54865853ebc1f901964e25a2e7a0e4d2cb6b9648",
Sum: "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=",
},
"github.com/prometheus/prometheus/documentation/prometheus-mixin": {
LegacyNameCompat: "prometheus",
Source: deps.Source{
GitSource: &deps.Git{
Scheme: deps.GitSchemeHTTPS,
Host: "github.com",
User: "prometheus",
Repo: "prometheus",
Subdir: "/documentation/prometheus-mixin",
},
},
Version: "7c039a6b3b4b2a9d7c613ac8bd3fc16e8ca79684",
Sum: "bVGOsq3hLOw2irNPAS91a5dZJqQlBUNWy3pVwM4+kIY=",
},
},
}
}
// TestUnmarshal checks that unmarshalling works
func TestUnmarshal(t *testing.T) {
var dst JsonnetFile
err := json.Unmarshal([]byte(jsonJF), &dst)
require.NoError(t, err)
assert.Equal(t, testData(), dst)
}
// TestMarshal checks that marshalling works
func TestMarshal(t *testing.T) {
data, err := json.Marshal(testData())
require.NoError(t, err)
assert.JSONEq(t, jsonJF, string(data))
}
// TestRemarshal checks that unmarshalling a previously marshalled object yields
// the same object
func TestRemarshal(t *testing.T) {
jf := testData()
data, err := json.Marshal(jf)
require.NoError(t, err)
var dst JsonnetFile
err = json.Unmarshal(data, &dst)
require.NoError(t, err)
assert.Equal(t, jf, dst)
}