feat(*): dryRun/debug mode/mode plugins/googleapis

Signed-off-by: storyicon <yuanchao@bilibili.com>

pkg/component/pluginmanager/git.go

@@ -0,0 +1,91 @@
+// Copyright 2021 storyicon@foxmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pluginmanager
+
+import (
+	"context"
+	"strings"
+
+	"github.com/storyicon/powerproto/pkg/util/command"
+	"github.com/storyicon/powerproto/pkg/util/logger"
+)
+
+// ErrGitList defines the git list error
+type ErrGitList struct {
+	*command.ErrCommandExec
+}
+
+// GetGitLatestCommitId is used to get the latest commit id
+func GetGitLatestCommitId(ctx context.Context, log logger.Logger, repo string) (string, error) {
+	data, err := command.Execute(ctx, log, "", "git", []string{
+		"ls-remote", repo, "HEAD",
+	}, nil)
+	if err != nil {
+		return "", &ErrGitList{
+			ErrCommandExec: err,
+		}
+	}
+	f := strings.Fields(string(data))
+	if len(f) != 2 {
+		return "", &ErrGitList{
+			ErrCommandExec: err,
+		}
+	}
+	return f[0], nil
+}
+
+// ListGitCommitIds is used to list git commit ids
+func ListGitCommitIds(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
+	data, err := command.Execute(ctx, log, "", "git", []string{
+		"ls-remote", repo,
+	}, nil)
+	if err != nil {
+		return nil, &ErrGitList{
+			ErrCommandExec: err,
+		}
+	}
+	var commitIds []string
+	for _, line := range strings.Split(string(data), "\n") {
+		f := strings.Fields(line)
+		if len(f) != 2 {
+			continue
+		}
+		commitIds = append(commitIds, f[0])
+	}
+	return commitIds, nil
+}
+
+// ListGitTags is used to list the git tags of specified repository
+func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
+	data, err := command.Execute(ctx, log, "", "git", []string{
+		"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
+	}, nil)
+	if err != nil {
+		return nil, &ErrGitList{
+			ErrCommandExec: err,
+		}
+	}
+	var tags []string
+	for _, line := range strings.Split(string(data), "\n") {
+		f := strings.Fields(line)
+		if len(f) != 2 {
+			continue
+		}
+		if strings.HasPrefix(f[1], "refs/tags/") {
+			tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
+		}
+	}
+	return tags, nil
+}

pkg/component/pluginmanager/googleapis.go

@@ -0,0 +1,81 @@
+// Copyright 2021 storyicon@foxmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pluginmanager
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/mholt/archiver"
+)
+
+// GoogleAPIRelease defines the release of google api
+type GoogleAPIRelease struct {
+	workspace string
+	commit    string
+}
+
+// GetDir is used to get the dir of google api
+func (p *GoogleAPIRelease) GetDir() string {
+	return filepath.Join(p.workspace, "googleapis-"+p.commit)
+}
+
+// Clear is used to clear the workspace
+func (p *GoogleAPIRelease) Clear() error {
+	return os.RemoveAll(p.workspace)
+}
+
+// GetGoogleAPIRelease is used to get the release of google api
+func GetGoogleAPIRelease(ctx context.Context, commitId string) (*GoogleAPIRelease, error) {
+	filename := fmt.Sprintf("%s.zip", commitId)
+	uri := fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s", filename)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
+	if err != nil {
+		return nil, &ErrHTTPDownload{
+			Url: uri,
+			Err: err,
+		}
+	}
+	workspace, err := os.MkdirTemp("", "")
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, &ErrHTTPDownload{
+			Url: uri,
+			Err: err,
+		}
+	}
+	zipFilePath := filepath.Join(workspace, filename)
+	if err := downloadFile(resp, zipFilePath); err != nil {
+		return nil, &ErrHTTPDownload{
+			Url:  uri,
+			Err:  err,
+			Code: resp.StatusCode,
+		}
+	}
+	zip := archiver.NewZip()
+	if err := zip.Unarchive(zipFilePath, workspace); err != nil {
+		return nil, err
+	}
+	return &GoogleAPIRelease{
+		commit:    commitId,
+		workspace: workspace,
+	}, nil
+}

pkg/component/pluginmanager/manager.go

@@ -41,7 +41,17 @@ type PluginManager interface {
 	// InstallPlugin is used to install plugin
 	InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
 
-	// GetProtocLatestVersion is used to geet the latest version of protoc
+	// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
+	GetGoogleAPIsLatestVersion(ctx context.Context) (string, error)
+	// InstallGoogleAPIs is used to install google apis
+	InstallGoogleAPIs(ctx context.Context, commitId string) (local string, err error)
+	// ListGoogleAPIsVersions is used to list protoc version
+	ListGoogleAPIsVersions(ctx context.Context) ([]string, error)
+	// IsGoogleAPIsInstalled is used to check whether the protoc is installed
+	IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error)
+	// GoogleAPIsPath  returns the googleapis path
+	GoogleAPIsPath(ctx context.Context, commitId string) string
+	// GetProtocLatestVersion is used to get the latest version of protoc
 	GetProtocLatestVersion(ctx context.Context) (string, error)
 	// ListProtocVersions is used to list protoc version
 	ListProtocVersions(ctx context.Context) ([]string, error)
@@ -130,6 +140,82 @@ func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, ver
 	return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
 }
 
+// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
+func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (string, error) {
+	versions, err := b.ListGoogleAPIsVersions(ctx)
+	if err != nil {
+		return "", err
+	}
+	if len(versions) == 0 {
+		return "", errors.New("no version list")
+	}
+	return versions[len(versions)-1], nil
+}
+
+// InstallGoogleAPIs is used to install google apis
+func (b *BasicPluginManager) InstallGoogleAPIs(ctx context.Context, commitId string) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
+	defer cancel()
+	local := PathForGoogleAPIs(b.storageDir, commitId)
+	exists, err := util.IsDirExists(local)
+	if err != nil {
+		return "", err
+	}
+	if exists {
+		return local, nil
+	}
+	release, err := GetGoogleAPIRelease(ctx, commitId)
+	if err != nil {
+		return "", err
+	}
+	defer release.Clear()
+	if err := util.CopyDirectory(release.GetDir(), local); err != nil {
+		return "", err
+	}
+	return local, nil
+}
+
+// ListGoogleAPIsVersions is used to list protoc version
+func (b *BasicPluginManager) ListGoogleAPIsVersions(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
+	defer cancel()
+
+	b.versionsLock.RLock()
+	versions, ok := b.versions["googleapis"]
+	b.versionsLock.RUnlock()
+	if ok {
+		return versions, nil
+	}
+	versions, err := ListGitCommitIds(ctx, b.Logger, consts.GoogleAPIsRepository)
+	if err != nil {
+		return nil, err
+	}
+	b.versionsLock.Lock()
+	b.versions["googleapis"] = versions
+	b.versionsLock.Unlock()
+	return versions, nil
+}
+
+// IsGoogleAPIsInstalled is used to check whether the protoc is installed
+func (b *BasicPluginManager) IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error) {
+	local := PathForGoogleAPIs(b.storageDir, commitId)
+	exists, err := util.IsDirExists(local)
+	return exists, local, err
+}
+
+// GoogleAPIsPath returns the googleapis path
+func (b *BasicPluginManager) GoogleAPIsPath(ctx context.Context, commitId string) string {
+	return PathForGoogleAPIs(b.storageDir, commitId)
+}
+
+// IsProtocInstalled is used to check whether the protoc is installed
+func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
+	if strings.HasPrefix(version, "v") {
+		version = strings.TrimPrefix(version, "v")
+	}
+	return IsProtocInstalled(ctx, b.storageDir, version)
+}
+
 // GetProtocLatestVersion is used to geet the latest version of protoc
 func (b *BasicPluginManager) GetProtocLatestVersion(ctx context.Context) (string, error) {
 	versions, err := b.ListProtocVersions(ctx)
@@ -163,14 +249,6 @@ func (b *BasicPluginManager) ListProtocVersions(ctx context.Context) ([]string,
 	return versions, nil
 }
 
-// IsProtocInstalled is used to check whether the protoc is installed
-func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
-	if strings.HasPrefix(version, "v") {
-		version = strings.TrimPrefix(version, "v")
-	}
-	return IsProtocInstalled(ctx, b.storageDir, version)
-}
-
 // InstallProtoc is used to install protoc of specified version
 func (b *BasicPluginManager) InstallProtoc(ctx context.Context, version string) (string, error) {
 	ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)

pkg/component/pluginmanager/paths.go

@@ -23,7 +23,6 @@ import (
 	"github.com/storyicon/powerproto/pkg/util"
 )
 
-
 // PathForInclude is used to get the local directory of include files
 func PathForInclude(storageDir string) string {
 	return filepath.Join(storageDir, "include")
@@ -47,6 +46,11 @@ func GetPluginPath(path string, version string) (string, error) {
 	return filepath.Join(enc + "@" + encVer), nil
 }
 
+// PathForGoogleAPIs is used to get the google apis path
+func PathForGoogleAPIs(storageDir string, commitId string) string {
+	return filepath.Join(storageDir, "googleapis", commitId)
+}
+
 // PathForPluginDir is used to get the local directory where the specified version plug-in should be stored
 func PathForPluginDir(storageDir string, path string, version string) (string, error) {
 	pluginPath, err := GetPluginPath(path, version)

pkg/component/pluginmanager/pluginmanager_test.go

@@ -70,4 +70,22 @@ var _ = Describe("Pluginmanager", func() {
 		Expect(exists).To(BeTrue())
 		Expect(len(local) != 0).To(BeTrue())
 	})
+	It("should able to install googleapis", func() {
+		versions, err := manager.ListGoogleAPIsVersions(context.TODO())
+		Expect(err).To(BeNil())
+		Expect(len(versions) > 0).To(BeTrue())
+
+		latestVersion, err := manager.GetGoogleAPIsLatestVersion(context.TODO())
+		Expect(err).To(BeNil())
+		Expect(latestVersion).To(Equal(versions[len(versions)-1]))
+
+		local, err := manager.InstallGoogleAPIs(context.TODO(), latestVersion)
+		Expect(err).To(BeNil())
+		Expect(len(local) > 0).To(BeTrue())
+
+		exists, local, err := manager.IsGoogleAPIsInstalled(context.TODO(), latestVersion)
+		Expect(err).To(BeNil())
+		Expect(exists).To(BeTrue())
+		Expect(len(local) != 0).To(BeTrue())
+	})
 })

pkg/component/pluginmanager/protoc.go

@@ -29,8 +29,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/storyicon/powerproto/pkg/util"
-	"github.com/storyicon/powerproto/pkg/util/command"
-	"github.com/storyicon/powerproto/pkg/util/logger"
 )
 
 // ProtocRelease defines the release of protoc
@@ -107,34 +105,6 @@ func IsProtocInstalled(ctx context.Context, storageDir string, version string) (
 	return exists, local, nil
 }
 
-// ErrGitList defines the git list error
-type ErrGitList struct {
-	*command.ErrCommandExec
-}
-
-// ListGitTags is used to list the git tags of specified repository
-func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
-	data, err := command.Execute(ctx, log, "", "git", []string{
-		"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
-	}, nil)
-	if err != nil {
-		return nil, &ErrGitList{
-			ErrCommandExec: err,
-		}
-	}
-	var tags []string
-	for _, line := range strings.Split(string(data), "\n") {
-		f := strings.Fields(line)
-		if len(f) != 2 {
-			continue
-		}
-		if strings.HasPrefix(f[1], "refs/tags/") {
-			tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
-		}
-	}
-	return tags, nil
-}
-
 func inferProtocReleaseSuffix() (string, error) {
 	goos := strings.ToLower(runtime.GOOS)
 	arch := strings.ToLower(runtime.GOARCH)