technofab/powerproto
1
0
Fork 0
mirror of https://github.com/TECHNOFAB11/powerproto.git synced 2025-12-11 23:50:04 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(*): dryRun/debug mode/mode plugins/googleapis

Browse source 
Signed-off-by: storyicon <yuanchao@bilibili.com>
This commit is contained in:
storyicon 2021-07-21 20:25:38 +08:00
parent 9aac714c32
commit da77c8086d
No known key found for this signature in database
GPG key ID: 245915D985F966CF
26 changed files with 730 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

11
pkg/component/actionmanager/actions/copy.go
View file

@ -20,8 +20,8 @@ import (
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -48,15 +48,20 @@ func ActionCopy(ctx context.Context, log logger.Logger, args []string, options *
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
if consts.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "copy",
"from": absSource,
"to": absDestination,
}, "DryRun")
}, consts.TextDryRun)
return nil
}
log.LogDebug(map[string]interface{}{
"action": "copy",
"from": absSource,
"to": absDestination,
}, consts.TextExecuteAction)
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}

12
pkg/component/actionmanager/actions/move.go
View file

@ -21,8 +21,8 @@ import (
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -49,15 +49,21 @@ func ActionMove(ctx context.Context, log logger.Logger, args []string, options *
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
if consts.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "move",
"from": absSource,
"to": absDestination,
}, "DryRun")
}, consts.TextDryRun)
return nil
}
log.LogDebug(map[string]interface{}{
"action": "move",
"from": absSource,
"to": absDestination,
}, consts.TextExecuteAction)
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}

11
pkg/component/actionmanager/actions/remove.go
View file

@ -21,7 +21,7 @@ import (
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -35,14 +35,19 @@ func ActionRemove(ctx context.Context, log logger.Logger, args []string, options
}
path := filepath.Join(filepath.Dir(options.ConfigFilePath), arg)
if command.IsDryRun(ctx) {
if consts.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "remove",
"target": path,
}, "DryRun")
}, consts.TextDryRun)
return nil
}
log.LogDebug(map[string]interface{}{
"action": "remove",
"target": path,
}, consts.TextExecuteAction)
if err := os.RemoveAll(path); err != nil {
return err
}

12
pkg/component/actionmanager/actions/replace.go
View file

@ -23,8 +23,8 @@ import (
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -59,15 +59,21 @@ func ActionReplace(ctx context.Context, log logger.Logger, args []string, option
panic(err)
}
if matched {
if command.IsDryRun(ctx) {
if consts.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "replace",
"file": path,
"from": from,
"to": to,
}, "DryRun")
}, consts.TextDryRun)
return nil
}
log.LogDebug(map[string]interface{}{
"action": "replace",
"file": path,
"from": from,
"to": to,
}, consts.TextExecuteAction)
data, err := ioutil.ReadFile(path)
if err != nil {
return err

54
pkg/component/compilermanager/compiler.go
View file

@ -45,9 +45,10 @@ type BasicCompiler struct {
config configs.ConfigItem
pluginManager pluginmanager.PluginManager
protocPath string
arguments []string
dir string
protocPath string
arguments []string
googleapisPath string
dir string
}
// NewCompiler is used to create a compiler
@ -72,6 +73,9 @@ func NewBasicCompiler(
config: config,
pluginManager: pluginManager,
}
if err := basic.calcGoogleAPIs(ctx); err != nil {
return nil, err
}
if err := basic.calcProto(ctx); err != nil {
return nil, err
}
@ -84,11 +88,17 @@ func NewBasicCompiler(
return basic, nil
}
// Compile is used to compile proto file
func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error {
arguments := make([]string, len(b.arguments))
copy(arguments, b.arguments)
// contains source relative
if util.Contains(b.config.Config().ImportPaths,
consts.KeySourceRelative) {
arguments = append(arguments, "--proto_path="+filepath.Dir(protoFilePath))
}
arguments = append(arguments, protoFilePath)
_, err := command.Execute(ctx,
b.Logger, b.dir, b.protocPath, arguments, nil)
@ -118,6 +128,30 @@ func (b *BasicCompiler) calcDir(ctx context.Context) error {
return nil
}
func (b *BasicCompiler) calcGoogleAPIs(ctx context.Context) error {
cfg := b.config.Config()
commitId := cfg.GoogleAPIs
if commitId == "" {
return nil
}
if !util.Contains(cfg.ImportPaths, consts.KeyPowerProtoGoogleAPIs) {
return nil
}
if commitId == "latest" {
latestVersion, err := b.pluginManager.GetGoogleAPIsLatestVersion(ctx)
if err != nil {
return err
}
commitId = latestVersion
}
local, err := b.pluginManager.InstallGoogleAPIs(ctx, commitId)
if err != nil {
return err
}
b.googleapisPath = local
return nil
}
func (b *BasicCompiler) calcProto(ctx context.Context) error {
cfg := b.config
protocVersion := cfg.Config().Protoc
@ -143,9 +177,17 @@ func (b *BasicCompiler) calcArguments(ctx context.Context) error {
dir := filepath.Dir(cfg.Path())
// build import paths
Loop:
for _, path := range cfg.Config().ImportPaths {
if path == consts.KeyPowerProtoInclude {
switch path {
case consts.KeyPowerProtoInclude:
path = b.pluginManager.IncludePath(ctx)
case consts.KeyPowerProtoGoogleAPIs:
if b.googleapisPath != "" {
path = b.googleapisPath
}
case consts.KeySourceRelative:
continue Loop
}
path = util.RenderPathWithEnv(path)
if !filepath.IsAbs(path) {
@ -176,4 +218,4 @@ func (b *BasicCompiler) calcArguments(ctx context.Context) error {
}
b.arguments = arguments
return nil
}
}

91
pkg/component/pluginmanager/git.go Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"strings"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ErrGitList defines the git list error
type ErrGitList struct {
*command.ErrCommandExec
}
// GetGitLatestCommitId is used to get the latest commit id
func GetGitLatestCommitId(ctx context.Context, log logger.Logger, repo string) (string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", repo, "HEAD",
}, nil)
if err != nil {
return "", &ErrGitList{
ErrCommandExec: err,
}
}
f := strings.Fields(string(data))
if len(f) != 2 {
return "", &ErrGitList{
ErrCommandExec: err,
}
}
return f[0], nil
}
// ListGitCommitIds is used to list git commit ids
func ListGitCommitIds(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", repo,
}, nil)
if err != nil {
return nil, &ErrGitList{
ErrCommandExec: err,
}
}
var commitIds []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
commitIds = append(commitIds, f[0])
}
return commitIds, nil
}
// ListGitTags is used to list the git tags of specified repository
func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
}, nil)
if err != nil {
return nil, &ErrGitList{
ErrCommandExec: err,
}
}
var tags []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
if strings.HasPrefix(f[1], "refs/tags/") {
tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
}
}
return tags, nil
}

81
pkg/component/pluginmanager/googleapis.go Normal file
View file

@ -0,0 +1,81 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"github.com/mholt/archiver"
)
// GoogleAPIRelease defines the release of google api
type GoogleAPIRelease struct {
workspace string
commit string
}
// GetDir is used to get the dir of google api
func (p *GoogleAPIRelease) GetDir() string {
return filepath.Join(p.workspace, "googleapis-"+p.commit)
}
// Clear is used to clear the workspace
func (p *GoogleAPIRelease) Clear() error {
return os.RemoveAll(p.workspace)
}
// GetGoogleAPIRelease is used to get the release of google api
func GetGoogleAPIRelease(ctx context.Context, commitId string) (*GoogleAPIRelease, error) {
filename := fmt.Sprintf("%s.zip", commitId)
uri := fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s", filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &GoogleAPIRelease{
commit: commitId,
workspace: workspace,
}, nil
}

96
pkg/component/pluginmanager/manager.go
View file

@ -41,7 +41,17 @@ type PluginManager interface {
// InstallPlugin is used to install plugin
InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
// GetProtocLatestVersion is used to geet the latest version of protoc
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
GetGoogleAPIsLatestVersion(ctx context.Context) (string, error)
// InstallGoogleAPIs is used to install google apis
InstallGoogleAPIs(ctx context.Context, commitId string) (local string, err error)
// ListGoogleAPIsVersions is used to list protoc version
ListGoogleAPIsVersions(ctx context.Context) ([]string, error)
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error)
// GoogleAPIsPath returns the googleapis path
GoogleAPIsPath(ctx context.Context, commitId string) string
// GetProtocLatestVersion is used to get the latest version of protoc
GetProtocLatestVersion(ctx context.Context) (string, error)
// ListProtocVersions is used to list protoc version
ListProtocVersions(ctx context.Context) ([]string, error)
@ -130,6 +140,82 @@ func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, ver
return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
}
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListGoogleAPIsVersions(ctx)
if err != nil {
return "", err
}
if len(versions) == 0 {
return "", errors.New("no version list")
}
return versions[len(versions)-1], nil
}
// InstallGoogleAPIs is used to install google apis
func (b *BasicPluginManager) InstallGoogleAPIs(ctx context.Context, commitId string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
if err != nil {
return "", err
}
if exists {
return local, nil
}
release, err := GetGoogleAPIRelease(ctx, commitId)
if err != nil {
return "", err
}
defer release.Clear()
if err := util.CopyDirectory(release.GetDir(), local); err != nil {
return "", err
}
return local, nil
}
// ListGoogleAPIsVersions is used to list protoc version
func (b *BasicPluginManager) ListGoogleAPIsVersions(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions["googleapis"]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListGitCommitIds(ctx, b.Logger, consts.GoogleAPIsRepository)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions["googleapis"] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error) {
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
return exists, local, err
}
// GoogleAPIsPath returns the googleapis path
func (b *BasicPluginManager) GoogleAPIsPath(ctx context.Context, commitId string) string {
return PathForGoogleAPIs(b.storageDir, commitId)
}
// IsProtocInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
return IsProtocInstalled(ctx, b.storageDir, version)
}
// GetProtocLatestVersion is used to geet the latest version of protoc
func (b *BasicPluginManager) GetProtocLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListProtocVersions(ctx)
@ -163,14 +249,6 @@ func (b *BasicPluginManager) ListProtocVersions(ctx context.Context) ([]string,
return versions, nil
}
// IsProtocInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
return IsProtocInstalled(ctx, b.storageDir, version)
}
// InstallProtoc is used to install protoc of specified version
func (b *BasicPluginManager) InstallProtoc(ctx context.Context, version string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)

6
pkg/component/pluginmanager/paths.go
View file

@ -23,7 +23,6 @@ import (
"github.com/storyicon/powerproto/pkg/util"
)
// PathForInclude is used to get the local directory of include files
func PathForInclude(storageDir string) string {
return filepath.Join(storageDir, "include")
@ -47,6 +46,11 @@ func GetPluginPath(path string, version string) (string, error) {
return filepath.Join(enc + "@" + encVer), nil
}
// PathForGoogleAPIs is used to get the google apis path
func PathForGoogleAPIs(storageDir string, commitId string) string {
return filepath.Join(storageDir, "googleapis", commitId)
}
// PathForPluginDir is used to get the local directory where the specified version plug-in should be stored
func PathForPluginDir(storageDir string, path string, version string) (string, error) {
pluginPath, err := GetPluginPath(path, version)

18
pkg/component/pluginmanager/pluginmanager_test.go
View file

@ -70,4 +70,22 @@ var _ = Describe("Pluginmanager", func() {
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
It("should able to install googleapis", func() {
versions, err := manager.ListGoogleAPIsVersions(context.TODO())
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetGoogleAPIsLatestVersion(context.TODO())
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallGoogleAPIs(context.TODO(), latestVersion)
Expect(err).To(BeNil())
Expect(len(local) > 0).To(BeTrue())
exists, local, err := manager.IsGoogleAPIsInstalled(context.TODO(), latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
})

30
pkg/component/pluginmanager/protoc.go
View file

@ -29,8 +29,6 @@ import (
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ProtocRelease defines the release of protoc
@ -107,34 +105,6 @@ func IsProtocInstalled(ctx context.Context, storageDir string, version string) (
return exists, local, nil
}
// ErrGitList defines the git list error
type ErrGitList struct {
*command.ErrCommandExec
}
// ListGitTags is used to list the git tags of specified repository
func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
}, nil)
if err != nil {
return nil, &ErrGitList{
ErrCommandExec: err,
}
}
var tags []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
if strings.HasPrefix(f[1], "refs/tags/") {
tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
}
}
return tags, nil
}
func inferProtocReleaseSuffix() (string, error) {
goos := strings.ToLower(runtime.GOOS)
arch := strings.ToLower(runtime.GOARCH)