feat(*): support repositories

Signed-off-by: storyicon <yuanchao@bilibili.com>

pkg/bootstraps/build.go

@@ -67,54 +67,61 @@ func StepLookUpConfigs(
 	return configItems, nil
 }
 
-// StepInstallGoogleAPIs is used to install google apis
-func StepInstallGoogleAPIs(ctx context.Context,
+// StepInstallRepositories is used to install repositories
+func StepInstallRepositories(ctx context.Context,
 	pluginManager pluginmanager.PluginManager,
 	configItems []configs.ConfigItem) error {
 	deduplicate := map[string]struct{}{}
 	for _, config := range configItems {
-		version := config.Config().GoogleAPIs
-		if version != "" {
-			deduplicate[version] = struct{}{}
+		for _, pkg := range config.Config().Repositories {
+			deduplicate[pkg] = struct{}{}
 		}
 	}
 	if len(deduplicate) == 0 {
 		return nil
 	}
-
 	progress := progressbar.GetProgressBar(ctx, len(deduplicate))
-	progress.SetPrefix("Install googleapis")
+	progress.SetPrefix("Install repositories")
 
-	versionsMap := map[string]struct{}{}
-	for version := range deduplicate {
+	repoMap := map[string]struct{}{}
+	for pkg := range deduplicate {
+		path, version, ok := util.SplitGoPackageVersion(pkg)
+		if !ok {
+			return errors.Errorf("invalid format: %s, should in path@version format", pkg)
+		}
 		if version == "latest" {
-			progress.SetSuffix("query latest version of googleapis")
-			latestVersion, err := pluginManager.GetGoogleAPIsLatestVersion(ctx)
+			progress.SetSuffix("query latest version of %s", path)
+			latestVersion, err := pluginManager.GetGitRepoLatestVersion(ctx, path)
 			if err != nil {
-				return errors.Wrap(err, "failed to list googleapis versions")
+				return errors.Wrapf(err, "failed to query latest version of %s", path)
 			}
 			version = latestVersion
+			pkg = util.JoinGoPackageVersion(path, version)
 		}
-		progress.SetSuffix("check cache of googleapis %s", version)
-		exists, _, err := pluginManager.IsGoogleAPIsInstalled(ctx, version)
+		progress.SetSuffix("check cache of %s", pkg)
+		exists, _, err := pluginManager.IsGitRepoInstalled(ctx, path, version)
 		if err != nil {
 			return err
 		}
 		if exists {
-			progress.SetSuffix("the %s version of googleapis is already cached", version)
+			progress.SetSuffix("the %s version of %s is already cached", version, path)
 		} else {
-			progress.SetSuffix("install %s version of googleapis", version)
-			_, err = pluginManager.InstallGoogleAPIs(ctx, version)
+			progress.SetSuffix("install %s version of %s", version, path)
+			_, err = pluginManager.InstallGitRepo(ctx, path, version)
 			if err != nil {
 				return err
 			}
-			progress.SetSuffix("the %s version of googleapis is installed", version)
+			progress.SetSuffix("the %s version of %s is installed", version, path)
 		}
-		versionsMap[version] = struct{}{}
+		repoMap[pkg] = struct{}{}
 		progress.Incr()
 	}
+	progress.SetSuffix("all repositories have been installed")
 	progress.Wait()
-	fmt.Println("the following versions of googleapis will be used:", util.SetToSlice(versionsMap))
+	fmt.Println("the following versions of googleapis will be used:")
+	for pkg := range repoMap {
+		fmt.Printf("	%s\r\n", pkg)
+	}
 	return nil
 }
 
@@ -320,7 +327,7 @@ func Compile(ctx context.Context, targets []string) error {
 	if err := StepInstallProtoc(ctx, pluginManager, configItems); err != nil {
 		return err
 	}
-	if err := StepInstallGoogleAPIs(ctx, pluginManager, configItems); err != nil {
+	if err := StepInstallRepositories(ctx, pluginManager, configItems); err != nil {
 		return err
 	}
 	if err := StepInstallPlugins(ctx, pluginManager, configItems); err != nil {

pkg/bootstraps/tidy.go

@@ -95,6 +95,21 @@ func StepTidyConfigFile(ctx context.Context,
 			item.Protoc = version
 			cleanable = true
 		}
+		for name, pkg := range item.Repositories {
+			path, version, ok := util.SplitGoPackageVersion(pkg)
+			if !ok {
+				return errors.Errorf("invalid package format: %s, should be in path@version format", pkg)
+			}
+			if version == "latest" {
+				progress.SetSuffix("query latest version of %s", path)
+				version, err := pluginManager.GetGitRepoLatestVersion(ctx, path)
+				if err != nil {
+					return err
+				}
+				item.Repositories[name] = util.JoinGoPackageVersion(path, version)
+				cleanable = true
+			}
+		}
 		for name, pkg := range item.Plugins {
 			path, version, ok := util.SplitGoPackageVersion(pkg)
 			if !ok {

pkg/component/compilermanager/compiler.go

@@ -45,10 +45,11 @@ type BasicCompiler struct {
 	config        configs.ConfigItem
 	pluginManager pluginmanager.PluginManager
 
-	protocPath     string
-	arguments      []string
-	googleapisPath string
-	dir            string
+	protocPath string
+	arguments  []string
+	// map[name]local
+	repositories map[string]string
+	dir          string
 }
 
 // NewCompiler is used to create a compiler
@@ -73,7 +74,10 @@ func NewBasicCompiler(
 		config:        config,
 		pluginManager: pluginManager,
 	}
-	if err := basic.calcGoogleAPIs(ctx); err != nil {
+	basic.repositories = map[string]string{
+		consts.KeyNamePowerProtocInclude: basic.pluginManager.IncludePath(ctx),
+	}
+	if err := basic.calcRepositories(ctx); err != nil {
 		return nil, err
 	}
 	if err := basic.calcProto(ctx); err != nil {
@@ -98,7 +102,7 @@ func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error
 		consts.KeySourceRelative) {
 		arguments = append(arguments, "--proto_path="+filepath.Dir(protoFilePath))
 	}
-
+	arguments = util.DeduplicateSliceStably(arguments)
 	arguments = append(arguments, protoFilePath)
 	_, err := command.Execute(ctx,
 		b.Logger, b.dir, b.protocPath, arguments, nil)
@@ -117,7 +121,7 @@ func (b *BasicCompiler) GetConfig(ctx context.Context) configs.ConfigItem {
 
 func (b *BasicCompiler) calcDir(ctx context.Context) error {
 	if dir := b.config.Config().ProtocWorkDir; dir != "" {
-		dir = util.RenderPathWithEnv(dir)
+		dir = util.RenderPathWithEnv(dir, nil)
 		if !filepath.IsAbs(dir) {
 			dir = filepath.Join(b.config.Path(), dir)
 		}
@@ -128,30 +132,6 @@ func (b *BasicCompiler) calcDir(ctx context.Context) error {
 	return nil
 }
 
-func (b *BasicCompiler) calcGoogleAPIs(ctx context.Context) error {
-	cfg := b.config.Config()
-	commitId := cfg.GoogleAPIs
-	if commitId == "" {
-		return nil
-	}
-	if !util.Contains(cfg.ImportPaths, consts.KeyPowerProtoGoogleAPIs) {
-		return nil
-	}
-	if commitId == "latest" {
-		latestVersion, err := b.pluginManager.GetGoogleAPIsLatestVersion(ctx)
-		if err != nil {
-			return err
-		}
-		commitId = latestVersion
-	}
-	local, err := b.pluginManager.InstallGoogleAPIs(ctx, commitId)
-	if err != nil {
-		return err
-	}
-	b.googleapisPath = local
-	return nil
-}
-
 func (b *BasicCompiler) calcProto(ctx context.Context) error {
 	cfg := b.config
 	protocVersion := cfg.Config().Protoc
@@ -179,17 +159,10 @@ func (b *BasicCompiler) calcArguments(ctx context.Context) error {
 	// build import paths
 Loop:
 	for _, path := range cfg.Config().ImportPaths {
-		switch path {
-		case consts.KeyPowerProtoInclude:
-			path = b.pluginManager.IncludePath(ctx)
-		case consts.KeyPowerProtoGoogleAPIs:
-			if b.googleapisPath != "" {
-				path = b.googleapisPath
-			}
-		case consts.KeySourceRelative:
+		if path == consts.KeySourceRelative {
 			continue Loop
 		}
-		path = util.RenderPathWithEnv(path)
+		path = util.RenderPathWithEnv(path, b.repositories)
 		if !filepath.IsAbs(path) {
 			path = filepath.Join(dir, path)
 		}
@@ -219,3 +192,26 @@ Loop:
 	b.arguments = arguments
 	return nil
 }
+
+func (b *BasicCompiler) calcRepositories(ctx context.Context) error {
+	cfg := b.config
+	for name, pkg := range cfg.Config().Repositories {
+		path, version, ok := util.SplitGoPackageVersion(pkg)
+		if !ok {
+			return errors.Errorf("failed to parse: %s", pkg)
+		}
+		if version == "latest" {
+			latestVersion, err := b.pluginManager.GetGitRepoLatestVersion(ctx, path)
+			if err != nil {
+				return err
+			}
+			version = latestVersion
+		}
+		local, err := b.pluginManager.InstallGitRepo(ctx, path, version)
+		if err != nil {
+			return errors.Wrap(err, "failed to get plugin path")
+		}
+		b.repositories[name] = local
+	}
+	return nil
+}

pkg/component/pluginmanager/git.go

@@ -16,8 +16,15 @@ package pluginmanager
 
 import (
 	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
+	"github.com/mholt/archiver"
+
 	"github.com/storyicon/powerproto/pkg/util/command"
 	"github.com/storyicon/powerproto/pkg/util/logger"
 )
@@ -89,3 +96,62 @@ func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string,
 	}
 	return tags, nil
 }
+
+// GithubArchive is github archive
+type GithubArchive struct {
+	uri       string
+	commit    string
+	workspace string
+}
+
+// GetGithubArchive is used to download github archive
+func GetGithubArchive(ctx context.Context, uri string, commitId string) (*GithubArchive, error) {
+	filename := fmt.Sprintf("%s.zip", commitId)
+	addr := fmt.Sprintf("%s/archive/%s", uri, filename)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, addr, nil)
+	if err != nil {
+		return nil, &ErrHTTPDownload{
+			Url: addr,
+			Err: err,
+		}
+	}
+	workspace, err := os.MkdirTemp("", "")
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, &ErrHTTPDownload{
+			Url: addr,
+			Err: err,
+		}
+	}
+	zipFilePath := filepath.Join(workspace, filename)
+	if err := downloadFile(resp, zipFilePath); err != nil {
+		return nil, &ErrHTTPDownload{
+			Url:  addr,
+			Err:  err,
+			Code: resp.StatusCode,
+		}
+	}
+	zip := archiver.NewZip()
+	if err := zip.Unarchive(zipFilePath, workspace); err != nil {
+		return nil, err
+	}
+	return &GithubArchive{
+		uri:       uri,
+		commit:    commitId,
+		workspace: workspace,
+	}, nil
+}
+
+// GetLocalDir is used to get local dir of archive
+func (c *GithubArchive) GetLocalDir() string {
+	dir := path.Base(c.uri) + "-" + c.commit
+	return filepath.Join(c.workspace, dir)
+}
+
+// Clear is used to clear the workspace
+func (c *GithubArchive) Clear() error {
+	return os.RemoveAll(c.workspace)
+}

pkg/component/pluginmanager/googleapis.go

@@ -1,81 +0,0 @@
-// Copyright 2021 storyicon@foxmail.com
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pluginmanager
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"path/filepath"
-
-	"github.com/mholt/archiver"
-)
-
-// GoogleAPIRelease defines the release of google api
-type GoogleAPIRelease struct {
-	workspace string
-	commit    string
-}
-
-// GetDir is used to get the dir of google api
-func (p *GoogleAPIRelease) GetDir() string {
-	return filepath.Join(p.workspace, "googleapis-"+p.commit)
-}
-
-// Clear is used to clear the workspace
-func (p *GoogleAPIRelease) Clear() error {
-	return os.RemoveAll(p.workspace)
-}
-
-// GetGoogleAPIRelease is used to get the release of google api
-func GetGoogleAPIRelease(ctx context.Context, commitId string) (*GoogleAPIRelease, error) {
-	filename := fmt.Sprintf("%s.zip", commitId)
-	uri := fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s", filename)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
-	if err != nil {
-		return nil, &ErrHTTPDownload{
-			Url: uri,
-			Err: err,
-		}
-	}
-	workspace, err := os.MkdirTemp("", "")
-	if err != nil {
-		return nil, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, &ErrHTTPDownload{
-			Url: uri,
-			Err: err,
-		}
-	}
-	zipFilePath := filepath.Join(workspace, filename)
-	if err := downloadFile(resp, zipFilePath); err != nil {
-		return nil, &ErrHTTPDownload{
-			Url:  uri,
-			Err:  err,
-			Code: resp.StatusCode,
-		}
-	}
-	zip := archiver.NewZip()
-	if err := zip.Unarchive(zipFilePath, workspace); err != nil {
-		return nil, err
-	}
-	return &GoogleAPIRelease{
-		commit:    commitId,
-		workspace: workspace,
-	}, nil
-}

pkg/component/pluginmanager/manager.go

@@ -41,16 +41,17 @@ type PluginManager interface {
 	// InstallPlugin is used to install plugin
 	InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
 
-	// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
-	GetGoogleAPIsLatestVersion(ctx context.Context) (string, error)
-	// InstallGoogleAPIs is used to install google apis
-	InstallGoogleAPIs(ctx context.Context, commitId string) (local string, err error)
-	// ListGoogleAPIsVersions is used to list protoc version
-	ListGoogleAPIsVersions(ctx context.Context) ([]string, error)
-	// IsGoogleAPIsInstalled is used to check whether the protoc is installed
-	IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error)
-	// GoogleAPIsPath  returns the googleapis path
-	GoogleAPIsPath(ctx context.Context, commitId string) string
+	// GetGitRepoLatestVersion is used to get the latest version of google apis
+	GetGitRepoLatestVersion(ctx context.Context, uri string) (string, error)
+	// InstallGitRepo is used to install google apis
+	InstallGitRepo(ctx context.Context, uri string, commitId string) (local string, err error)
+	// ListGitRepoVersions is used to list protoc version
+	ListGitRepoVersions(ctx context.Context, uri string) ([]string, error)
+	// IsGitRepoInstalled is used to check whether the protoc is installed
+	IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error)
+	// GitRepoPath  returns the googleapis path
+	GitRepoPath(ctx context.Context, commitId string) string
+
 	// GetProtocLatestVersion is used to get the latest version of protoc
 	GetProtocLatestVersion(ctx context.Context) (string, error)
 	// ListProtocVersions is used to list protoc version
@@ -140,9 +141,9 @@ func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, ver
 	return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
 }
 
-// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
-func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (string, error) {
-	versions, err := b.ListGoogleAPIsVersions(ctx)
+// GetGitRepoLatestVersion is used to get the latest version of google apis
+func (b *BasicPluginManager) GetGitRepoLatestVersion(ctx context.Context, url string) (string, error) {
+	versions, err := b.ListGitRepoVersions(ctx, url)
 	if err != nil {
 		return "", err
 	}
@@ -152,60 +153,67 @@ func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (st
 	return versions[len(versions)-1], nil
 }
 
-// InstallGoogleAPIs is used to install google apis
-func (b *BasicPluginManager) InstallGoogleAPIs(ctx context.Context, commitId string) (string, error) {
+// InstallGitRepo is used to install google apis
+func (b *BasicPluginManager) InstallGitRepo(ctx context.Context, uri string, commitId string) (string, error) {
 	ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
 	defer cancel()
-	local := PathForGoogleAPIs(b.storageDir, commitId)
-	exists, err := util.IsDirExists(local)
+	exists, local, err := b.IsGitRepoInstalled(ctx, uri, commitId)
 	if err != nil {
 		return "", err
 	}
 	if exists {
 		return local, nil
 	}
-	release, err := GetGoogleAPIRelease(ctx, commitId)
+	release, err := GetGithubArchive(ctx, uri, commitId)
 	if err != nil {
 		return "", err
 	}
 	defer release.Clear()
-	if err := util.CopyDirectory(release.GetDir(), local); err != nil {
+
+	codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
+	if err != nil {
+		return "", err
+	}
+	if err := util.CopyDirectory(release.GetLocalDir(), codePath); err != nil {
 		return "", err
 	}
 	return local, nil
 }
 
-// ListGoogleAPIsVersions is used to list protoc version
-func (b *BasicPluginManager) ListGoogleAPIsVersions(ctx context.Context) ([]string, error) {
+// ListGitRepoVersions is used to list protoc version
+func (b *BasicPluginManager) ListGitRepoVersions(ctx context.Context, uri string) ([]string, error) {
 	ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
 	defer cancel()
 
 	b.versionsLock.RLock()
-	versions, ok := b.versions["googleapis"]
+	versions, ok := b.versions[uri]
 	b.versionsLock.RUnlock()
 	if ok {
 		return versions, nil
 	}
-	versions, err := ListGitCommitIds(ctx, b.Logger, consts.GoogleAPIsRepository)
+	versions, err := ListGitCommitIds(ctx, b.Logger, uri)
 	if err != nil {
 		return nil, err
 	}
 	b.versionsLock.Lock()
-	b.versions["googleapis"] = versions
+	b.versions[uri] = versions
 	b.versionsLock.Unlock()
 	return versions, nil
 }
 
-// IsGoogleAPIsInstalled is used to check whether the protoc is installed
-func (b *BasicPluginManager) IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error) {
-	local := PathForGoogleAPIs(b.storageDir, commitId)
-	exists, err := util.IsDirExists(local)
-	return exists, local, err
+// IsGitRepoInstalled is used to check whether the protoc is installed
+func (b *BasicPluginManager) IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error) {
+	codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
+	if err != nil {
+		return false, "", err
+	}
+	exists, err := util.IsDirExists(codePath)
+	return exists, PathForGitRepos(b.storageDir, commitId), err
 }
 
-// GoogleAPIsPath returns the googleapis path
-func (b *BasicPluginManager) GoogleAPIsPath(ctx context.Context, commitId string) string {
-	return PathForGoogleAPIs(b.storageDir, commitId)
+// GitRepoPath returns the googleapis path
+func (b *BasicPluginManager) GitRepoPath(ctx context.Context, commitId string) string {
+	return PathForGitRepos(b.storageDir, commitId)
 }
 
 // IsProtocInstalled is used to check whether the protoc is installed

pkg/component/pluginmanager/paths.go

@@ -15,6 +15,7 @@
 package pluginmanager
 
 import (
+	"net/url"
 	"path"
 	"path/filepath"
 
@@ -46,9 +47,19 @@ func GetPluginPath(path string, version string) (string, error) {
 	return filepath.Join(enc + "@" + encVer), nil
 }
 
-// PathForGoogleAPIs is used to get the google apis path
-func PathForGoogleAPIs(storageDir string, commitId string) string {
-	return filepath.Join(storageDir, "googleapis", commitId)
+// PathForGitReposCode returns the code path for git repos
+func PathForGitReposCode(storageDir string, uri string, commitId string) (string, error) {
+	parsed, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+	dir := parsed.Host + parsed.Path
+	return filepath.Join(PathForGitRepos(storageDir, commitId), dir), nil
+}
+
+// PathForGitRepos is used to get the git repo local path
+func PathForGitRepos(storageDir string, commitId string) string {
+	return filepath.Join(storageDir, "gits", commitId)
 }
 
 // PathForPluginDir is used to get the local directory where the specified version plug-in should be stored

pkg/component/pluginmanager/pluginmanager_test.go

@@ -70,20 +70,20 @@ var _ = Describe("Pluginmanager", func() {
 		Expect(exists).To(BeTrue())
 		Expect(len(local) != 0).To(BeTrue())
 	})
-	It("should able to install googleapis", func() {
-		versions, err := manager.ListGoogleAPIsVersions(context.TODO())
+	It("should able to install git repos", func() {
+		const uri = "https://github.com/gogo/protobuf"
+		versions, err := manager.ListGitRepoVersions(context.TODO(), uri)
 		Expect(err).To(BeNil())
 		Expect(len(versions) > 0).To(BeTrue())
-
-		latestVersion, err := manager.GetGoogleAPIsLatestVersion(context.TODO())
+		latestVersion, err := manager.GetGitRepoLatestVersion(context.TODO(), uri)
 		Expect(err).To(BeNil())
 		Expect(latestVersion).To(Equal(versions[len(versions)-1]))
 
-		local, err := manager.InstallGoogleAPIs(context.TODO(), latestVersion)
+		local, err := manager.InstallGitRepo(context.TODO(), uri, latestVersion)
 		Expect(err).To(BeNil())
 		Expect(len(local) > 0).To(BeTrue())
 
-		exists, local, err := manager.IsGoogleAPIsInstalled(context.TODO(), latestVersion)
+		exists, local, err := manager.IsGitRepoInstalled(context.TODO(), uri, latestVersion)
 		Expect(err).To(BeNil())
 		Expect(exists).To(BeTrue())
 		Expect(len(local) != 0).To(BeTrue())

pkg/configs/configs.go

@@ -29,10 +29,10 @@ import (
 // Config defines the config model
 type Config struct {
 	Scopes        []string          `json:"scopes" yaml:"scopes"`
-	GoogleAPIs    string            `json:"googleapis" yaml:"googleapis"`
 	Protoc        string            `json:"protoc" yaml:"protoc"`
 	ProtocWorkDir string            `json:"protocWorkDir" yaml:"protocWorkDir"`
 	Plugins       map[string]string `json:"plugins" yaml:"plugins"`
+	Repositories  map[string]string `json:"repositories" yaml:"repositories"`
 	Options       []string          `json:"options" yaml:"options"`
 	ImportPaths   []string          `json:"importPaths" yaml:"importPaths"`
 	PostActions   []*PostAction     `json:"postActions" yaml:"postActions"`

pkg/consts/consts.go

@@ -26,11 +26,11 @@ import (
 // defines a set of const value
 const (
 	// ConfigFileName defines the config file name
-	ConfigFileName = "powerproto.yaml"
+	ConfigFileName            = "powerproto.yaml"
+	// KeyNamePowerProtocInclude is the key name of powerproto default include
+	KeyNamePowerProtocInclude = "POWERPROTO_INCLUDE"
 	// The default include can be referenced by this key in import paths
-	KeyPowerProtoInclude = "$POWERPROTO_INCLUDE"
-	// The googleapis can be referenced by this key in import paths
-	KeyPowerProtoGoogleAPIs = "$POWERPROTO_GOOGLEAPIS"
+	KeyPowerProtoInclude = "$" + KeyNamePowerProtocInclude
 	// KeySourceRelative can be specified in import paths to refer to
 	// the folder where the current proto file is located
 	KeySourceRelative = "$SOURCE_RELATIVE"

pkg/util/util.go

@@ -24,6 +24,19 @@ import (
 	"syscall"
 )
 
+// DeduplicateSlice is used to deduplicate slice items stably
+func DeduplicateSliceStably(items []string) []string {
+	data := make([]string, 0, len(items))
+	deduplicate := map[string]struct{}{}
+	for _, val := range items {
+		if _, exists := deduplicate[val]; !exists {
+			deduplicate[val] = struct{}{}
+			data = append(data, val)
+		}
+	}
+	return data
+}
+
 // ContainsEmpty is used to check whether items contains empty string
 func ContainsEmpty(items ...string) bool {
 	return Contains(items, "")
@@ -75,10 +88,17 @@ func GetExitCode(err error) int {
 var regexpEnvironmentVar = regexp.MustCompile(`\$[A-Za-z_]+`)
 
 // RenderPathWithEnv is used to render path with environment
-func RenderPathWithEnv(path string) string {
+func RenderPathWithEnv(path string, ext map[string]string) string {
 	matches := regexpEnvironmentVar.FindAllString(path, -1)
 	for _, match := range matches {
-		path = strings.ReplaceAll(path, match, os.Getenv(match[1:]))
+		key := match[1:]
+		val := ext[key]
+		if val == "" {
+			val = os.Getenv(key)
+		}
+		if val != "" {
+			path = strings.ReplaceAll(path, match, val)
+		}
 	}
 	return filepath.Clean(path)
 }

pkg/util/util_test.go

@@ -0,0 +1,87 @@
+// Copyright 2021 storyicon@foxmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func TestRenderPathWithEnv(t *testing.T) {
+	type args struct {
+		path string
+		ext  map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			args: args{
+				path: "$POWERPROTO_INCLUDE/protobuf",
+				ext: map[string]string{
+					"POWERPROTO_INCLUDE": "/mnt/powerproto/include",
+				},
+			},
+			want: filepath.Clean("/mnt/powerproto/include/protobuf"),
+		},
+		{
+			args: args{
+				path: "$POWERPROTO_INCLUDE/protobuf",
+			},
+			want: filepath.Clean("$POWERPROTO_INCLUDE/protobuf"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RenderPathWithEnv(tt.args.path, tt.args.ext); got != tt.want {
+				t.Errorf("RenderPathWithEnv() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDeduplicateSliceStably(t *testing.T) {
+	type args struct {
+		items []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			args: args{
+				items: []string{"a", "B", "c", "B"},
+			},
+			want: []string{"a", "B", "c"},
+		},
+		{
+			args: args{
+				items: []string{"B", "c", "B", "a"},
+			},
+			want: []string{"B", "c", "a"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := DeduplicateSliceStably(tt.args.items); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DeduplicateSliceStably() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}