feat: faster parser (#3)

* fix(load): support nested objects

* feat: initial golang parser

Moves most of load.libsonnet to a Golang app for hopefully much faster
parsing (fingers crossed)

* feat(fast): nested objects, subpackages

* feat: incorporate fast mode into main binary

Moves the work done in the fast directory into the main executable,
removing the Jsonnet based parser in favor of the much faster Golang one.

fast/main.go

@@ -0,0 +1,234 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/google/go-jsonnet"
+	"github.com/sh0rez/docsonnet/pkg/docsonnet"
+	"github.com/sh0rez/docsonnet/pkg/render"
+)
+
+func main() {
+	data, err := eval()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	// fmt.Println(string(data))
+
+	var d DS
+	if err := json.Unmarshal(data, &d); err != nil {
+		log.Fatalln(err)
+	}
+
+	pkg := load(d)
+
+	fmt.Println("render")
+	res := render.Render(pkg)
+	for k, v := range res {
+		fmt.Println(k)
+		if err := ioutil.WriteFile(filepath.Join("docs", k), []byte(v), 0644); err != nil {
+			log.Fatalln(err)
+		}
+	}
+}
+
+// load docsonnet
+//
+// Data assumptions:
+// - only map[string]interface{} and docsonnet fields
+// - docsonnet fields (#...) coming first
+func load(d DS) docsonnet.Package {
+	start := time.Now()
+
+	pkg := d.Package()
+	fmt.Println("load", pkg.Name)
+
+	pkg.API = make(docsonnet.Fields)
+	pkg.Sub = make(map[string]docsonnet.Package)
+
+	for k, v := range d {
+		if k == "#" {
+			continue
+		}
+
+		f := v.(map[string]interface{})
+
+		// docsonnet field
+		name := strings.TrimPrefix(k, "#")
+		if strings.HasPrefix(k, "#") {
+			pkg.API[name] = loadField(name, f, d)
+			continue
+		}
+
+		// non-docsonnet
+		// subpackage?
+		if _, ok := f["#"]; ok {
+			p := load(DS(f))
+			pkg.Sub[p.Name] = p
+			continue
+		}
+
+		// non-annotated nested?
+		// try to load, but skip when already loaded as annotated above
+		if nested, ok := loadNested(name, f); ok && !fieldsHas(pkg.API, name) {
+			pkg.API[name] = *nested
+			continue
+		}
+	}
+
+	fmt.Println("done load", pkg.Name, time.Since(start))
+	return pkg
+}
+
+func fieldsHas(f docsonnet.Fields, key string) bool {
+	_, b := f[key]
+	return b
+}
+
+func loadNested(name string, msi map[string]interface{}) (*docsonnet.Field, bool) {
+	out := docsonnet.Object{
+		Name:   name,
+		Fields: make(docsonnet.Fields),
+	}
+
+	ok := false
+	for k, v := range msi {
+		f := v.(map[string]interface{})
+		n := strings.TrimPrefix(k, "#")
+
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				out.Fields[n] = *l
+			}
+			continue
+		}
+
+		ok = true
+		l := loadField(n, f, msi)
+		out.Fields[n] = l
+	}
+
+	if !ok {
+		return nil, false
+	}
+
+	return &docsonnet.Field{Object: &out}, true
+}
+
+func loadField(name string, field map[string]interface{}, parent map[string]interface{}) docsonnet.Field {
+	if ifn, ok := field["function"]; ok {
+		return loadFn(name, ifn.(map[string]interface{}))
+	}
+
+	if iobj, ok := field["object"]; ok {
+		return loadObj(name, iobj.(map[string]interface{}), parent)
+	}
+
+	panic("docsonnet field lacking {function | object}")
+}
+
+func loadFn(name string, msi map[string]interface{}) docsonnet.Field {
+	fn := docsonnet.Function{
+		Name: name,
+		Help: msi["help"].(string),
+	}
+	if args, ok := msi["args"]; ok {
+		fn.Args = loadArgs(args.([]interface{}))
+	}
+	return docsonnet.Field{Function: &fn}
+}
+
+func loadArgs(is []interface{}) []docsonnet.Argument {
+	args := make([]docsonnet.Argument, len(is))
+	for i := range is {
+		arg := is[i].(map[string]interface{})
+		args[i] = docsonnet.Argument{
+			Name:    arg["name"].(string),
+			Type:    docsonnet.Type(arg["type"].(string)),
+			Default: arg["default"],
+		}
+	}
+	return args
+}
+
+func fieldNames(msi map[string]interface{}) []string {
+	out := make([]string, 0, len(msi))
+	for k := range msi {
+		out = append(out, k)
+	}
+	return out
+}
+
+func loadObj(name string, msi map[string]interface{}, parent map[string]interface{}) docsonnet.Field {
+	obj := docsonnet.Object{
+		Name:   name,
+		Help:   msi["help"].(string),
+		Fields: make(docsonnet.Fields),
+	}
+
+	// look for children in same key without #
+	var iChilds interface{}
+	var ok bool
+	if iChilds, ok = parent[name]; !ok {
+		fmt.Println("aborting, no", name, strings.Join(fieldNames(parent), ", "))
+		return docsonnet.Field{Object: &obj}
+	}
+
+	childs := iChilds.(map[string]interface{})
+	for k, v := range childs {
+		name := strings.TrimPrefix(k, "#")
+		f := v.(map[string]interface{})
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				obj.Fields[name] = *l
+			}
+			continue
+		}
+
+		obj.Fields[name] = loadField(name, f, childs)
+	}
+
+	return docsonnet.Field{Object: &obj}
+}
+
+type DS map[string]interface{}
+
+func (d DS) Package() docsonnet.Package {
+	hash, ok := d["#"]
+	if !ok {
+		log.Fatalln("Package declaration missing")
+	}
+
+	pkg := hash.(map[string]interface{})
+	return docsonnet.Package{
+		Help:   pkg["help"].(string),
+		Name:   pkg["name"].(string),
+		Import: pkg["import"].(string),
+	}
+}
+
+func eval() ([]byte, error) {
+	fmt.Println("eval start")
+	start := time.Now()
+
+	vm := jsonnet.MakeVM()
+	vm.Importer(&jsonnet.FileImporter{JPaths: []string{".."}})
+	data, err := ioutil.ReadFile("fast.libsonnet")
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := vm.EvaluateSnippet("fast.libsonnet", string(data))
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("eval:", time.Since(start))
+	return []byte(out), nil
+}

go.sum

@@ -2,8 +2,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/go-clix/cli v0.1.1 h1:T9N0AdMbmpFM9cLw42TcLL5sQ3YgyxTyHUhBK0GW1LI=
-github.com/go-clix/cli v0.1.1/go.mod h1:dYJevXraB9mXZFhz5clyQestG0qGcmT5rRC/P9etoRQ=
 github.com/go-clix/cli v0.1.2-0.20200502172020-b8f4629e879a h1:nh+UOawbjKgiUAJAgi8JHctNebEu6mjwDXsv8Xdln8w=
 github.com/go-clix/cli v0.1.2-0.20200502172020-b8f4629e879a/go.mod h1:dYJevXraB9mXZFhz5clyQestG0qGcmT5rRC/P9etoRQ=
 github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=

load.libsonnet

@@ -1,59 +1,24 @@
 local lib = {
-  // reshape converts the Jsonnet structure to the one used by docsonnet:
-  // - put fields into an `api` key
-  // - put subpackages into `sub` key
-  reshape(pkg)::
-    local aux(old, key) =
-      if key == '#' then
-        old
-      else if std.objectHas(pkg[key], '#') then
-        old { sub+: { [key]: $.package(pkg[key]) } }
-      else
-        old { api+: { [key]: pkg[key] } };
-
-    std.foldl(aux, std.objectFields(pkg), {})
-    + pkg['#'],
-
-  // fillObjects creates docsonnet objects from Jsonnet ones,
-  // also filling those that have been specified explicitely
-  fillObjects(api)::
+  scan(obj)::
     local aux(old, key) =
       if std.startsWith(key, '#') then
-        old { [key]: api[key] }
-      else if std.isObject(api[key]) && std.length(std.objectFields(api[key])) > 0 then
-        old { ['#' + key]+: { object+: {
-          fields: api[key],
-        } } }
+        true
+      else if std.isObject(obj[key]) then
+        old || $.scan(obj[key])
+      else old;
+    std.foldl(aux, std.objectFieldsAll(obj), false),
+
+  load(pkg)::
+    local aux(old, key) =
+      if !std.isObject(pkg[key]) then
+        old
+      else if std.startsWith(key, '#') then
+        old { [key]: pkg[key] }
+      else if self.scan(pkg[key]) then
+        old { [key]: $.load(pkg[key]) }
       else old;
 
-    std.foldl(aux, std.objectFields(api), {}),
-
-  // clean removes all hashes from field names
-  clean(api):: {
-    [std.lstripChars(key, '#')]:
-      if std.isObject(api[key]) then $.clean(api[key])
-      else api[key]
-    for key in std.objectFields(api)
-  },
-
-  cleanNonObj(api):: {
-    [key]:
-      if std.startsWith(key, "#") then api[key]
-      else if std.isObject(api[key]) then $.cleanNonObj(api[key])
-      else api[key]
-    for key in std.objectFieldsAll(api)
-    if std.isObject(api[key])
-  },
-
-  // package loads docsonnet from a Jsonnet package
-  package(pkg)::
-    local cleaned = self.cleanNonObj(pkg);
-    local reshaped = self.reshape(cleaned);
-    local filled =
-      if std.objectHas(reshaped, 'api')
-      then reshaped { api: $.fillObjects(reshaped.api) }
-      else reshaped;
-    self.clean(filled),
+    std.foldl(aux, std.objectFieldsAll(pkg), {}),
 };
 
-lib.package(std.extVar("main"))
+lib.load(std.extVar('main'))

pkg/docsonnet/fast.go

@@ -0,0 +1,183 @@
+package docsonnet
+
+import (
+	"fmt"
+	"log"
+	"strings"
+	"time"
+)
+
+// load docsonnet
+//
+// Data assumptions:
+// - only map[string]interface{} and fields
+// - fields (#...) coming first
+func fastLoad(d DS) Package {
+	start := time.Now()
+
+	pkg := d.Package()
+	fmt.Println("load", pkg.Name)
+
+	pkg.API = make(Fields)
+	pkg.Sub = make(map[string]Package)
+
+	for k, v := range d {
+		if k == "#" {
+			continue
+		}
+
+		f := v.(map[string]interface{})
+
+		// field
+		name := strings.TrimPrefix(k, "#")
+		if strings.HasPrefix(k, "#") {
+			pkg.API[name] = loadField(name, f, d)
+			continue
+		}
+
+		// non-docsonnet
+		// subpackage?
+		if _, ok := f["#"]; ok {
+			p := fastLoad(DS(f))
+			pkg.Sub[p.Name] = p
+			continue
+		}
+
+		// non-annotated nested?
+		// try to load, but skip when already loaded as annotated above
+		if nested, ok := loadNested(name, f); ok && !fieldsHas(pkg.API, name) {
+			pkg.API[name] = *nested
+			continue
+		}
+	}
+
+	fmt.Println("done load", pkg.Name, time.Since(start))
+	return pkg
+}
+
+func fieldsHas(f Fields, key string) bool {
+	_, b := f[key]
+	return b
+}
+
+func loadNested(name string, msi map[string]interface{}) (*Field, bool) {
+	out := Object{
+		Name:   name,
+		Fields: make(Fields),
+	}
+
+	ok := false
+	for k, v := range msi {
+		f := v.(map[string]interface{})
+		n := strings.TrimPrefix(k, "#")
+
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				out.Fields[n] = *l
+			}
+			continue
+		}
+
+		ok = true
+		l := loadField(n, f, msi)
+		out.Fields[n] = l
+	}
+
+	if !ok {
+		return nil, false
+	}
+
+	return &Field{Object: &out}, true
+}
+
+func loadField(name string, field map[string]interface{}, parent map[string]interface{}) Field {
+	if ifn, ok := field["function"]; ok {
+		return loadFn(name, ifn.(map[string]interface{}))
+	}
+
+	if iobj, ok := field["object"]; ok {
+		return loadObj(name, iobj.(map[string]interface{}), parent)
+	}
+
+	panic("field lacking {function | object}")
+}
+
+func loadFn(name string, msi map[string]interface{}) Field {
+	fn := Function{
+		Name: name,
+		Help: msi["help"].(string),
+	}
+	if args, ok := msi["args"]; ok {
+		fn.Args = loadArgs(args.([]interface{}))
+	}
+	return Field{Function: &fn}
+}
+
+func loadArgs(is []interface{}) []Argument {
+	args := make([]Argument, len(is))
+	for i := range is {
+		arg := is[i].(map[string]interface{})
+		args[i] = Argument{
+			Name:    arg["name"].(string),
+			Type:    Type(arg["type"].(string)),
+			Default: arg["default"],
+		}
+	}
+	return args
+}
+
+func fieldNames(msi map[string]interface{}) []string {
+	out := make([]string, 0, len(msi))
+	for k := range msi {
+		out = append(out, k)
+	}
+	return out
+}
+
+func loadObj(name string, msi map[string]interface{}, parent map[string]interface{}) Field {
+	obj := Object{
+		Name:   name,
+		Help:   msi["help"].(string),
+		Fields: make(Fields),
+	}
+
+	// look for children in same key without #
+	var iChilds interface{}
+	var ok bool
+	if iChilds, ok = parent[name]; !ok {
+		fmt.Println("aborting, no", name, strings.Join(fieldNames(parent), ", "))
+		return Field{Object: &obj}
+	}
+
+	childs := iChilds.(map[string]interface{})
+	for k, v := range childs {
+		name := strings.TrimPrefix(k, "#")
+		f := v.(map[string]interface{})
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				obj.Fields[name] = *l
+			}
+			continue
+		}
+
+		obj.Fields[name] = loadField(name, f, childs)
+	}
+
+	return Field{Object: &obj}
+}
+
+type DS map[string]interface{}
+
+func (d DS) Package() Package {
+	hash, ok := d["#"]
+	if !ok {
+		log.Fatalln("Package declaration missing")
+	}
+
+	pkg := hash.(map[string]interface{})
+	return Package{
+		Help:   pkg["help"].(string),
+		Name:   pkg["name"].(string),
+		Import: pkg["import"].(string),
+	}
+}

pkg/docsonnet/load.go

@@ -32,18 +32,22 @@ func Load(filename string) (*Package, error) {
 
 	// invoke load.libsonnet
 	vm.ExtCode("main", fmt.Sprintf(`(import "%s")`, filename))
+
+	log.Println("evaluating Jsonnet")
 	data, err := vm.EvaluateSnippet("load.libsonnet", string(load))
 	if err != nil {
 		return nil, err
 	}
 
+	log.Println("parsing result")
 	// parse the result
-	var d Package
+	var d DS
 	if err := json.Unmarshal([]byte(data), &d); err != nil {
 		log.Fatalln(err)
 	}
 
-	return &d, nil
+	p := fastLoad(d)
+	return &p, nil
 }
 
 // importer wraps jsonnet.FileImporter, to statically provide load.libsonnet,

pkg/render/render.go

@@ -15,6 +15,7 @@ func Render(pkg docsonnet.Package) map[string]string {
 }
 
 func render(pkg docsonnet.Package, parents []string, root bool) map[string]string {
+	fmt.Println("render", pkg.Name)
 	link := "/" + strings.Join(append(parents, pkg.Name), "/")
 	if root {
 		link = "/"