technofab/powerproto
1
0
Fork 0
mirror of https://github.com/TECHNOFAB11/powerproto.git synced 2026-02-02 09:25:07 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

init

Browse source 
Signed-off-by: storyicon <yuanchao@bilibili.com>
This commit is contained in:
storyicon 2021-07-21 00:24:43 +08:00
commit 9aac714c32
No known key found for this signature in database
GPG key ID: 245915D985F966CF
47 changed files with 5480 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

285
pkg/bootstraps/build.go Normal file
View file

@ -0,0 +1,285 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstraps
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/component/actionmanager"
"github.com/storyicon/powerproto/pkg/component/compilermanager"
"github.com/storyicon/powerproto/pkg/component/configmanager"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/concurrent"
"github.com/storyicon/powerproto/pkg/util/logger"
"github.com/storyicon/powerproto/pkg/util/progressbar"
)
// StepLookUpConfigs is used to lookup config files according to target proto files
func StepLookUpConfigs(
ctx context.Context,
targets []string,
configManager configmanager.ConfigManager,
) ([]configs.ConfigItem, error) {
progress := progressbar.GetProgressBar(len(targets))
progress.SetPrefix("Lookup configs of proto files")
var configItems []configs.ConfigItem
deduplicate := map[string]struct{}{}
for _, target := range targets {
cfg, err := configManager.GetConfig(ctx, target)
if err != nil {
return nil, err
}
if _, exists := deduplicate[cfg.ID()]; !exists {
configItems = append(configItems, cfg)
deduplicate[cfg.ID()] = struct{}{}
}
progress.SetSuffix("load %s", cfg.ID())
progress.Incr()
}
progress.SetSuffix("success!")
progress.Wait()
fmt.Printf("the following %d configurations will be used: \r\n", len(configItems))
for _, config := range configItems {
fmt.Printf(" %s \r\n", config.Path())
}
if len(configItems) == 0 {
return nil, errors.New("no config file matched, please check the scope of config file " +
"or use 'powerproto init' to create config file")
}
return configItems, nil
}
// StepInstallProtoc is used to install protoc
func StepInstallProtoc(ctx context.Context,
pluginManager pluginmanager.PluginManager,
configItems []configs.ConfigItem) error {
deduplicate := map[string]struct{}{}
for _, config := range configItems {
version := config.Config().Protoc
deduplicate[version] = struct{}{}
}
progress := progressbar.GetProgressBar(len(deduplicate))
progress.SetPrefix("Install protoc")
versionsMap := map[string]struct{}{}
for version := range deduplicate {
if version == "latest" {
progress.SetSuffix("query latest version of protoc")
latestVersion, err := pluginManager.GetProtocLatestVersion(ctx)
if err != nil {
return errors.Wrap(err, "failed to list protoc versions")
}
version = latestVersion
}
progress.SetSuffix("check cache of protoc %s", version)
exists, _, err := pluginManager.IsProtocInstalled(ctx, version)
if err != nil {
return err
}
if exists {
progress.SetSuffix("the %s version of protoc is already cached", version)
} else {
progress.SetSuffix("install %s version of protoc", version)
_, err = pluginManager.InstallProtoc(ctx, version)
if err != nil {
return err
}
progress.SetSuffix("the %s version of protoc is installed", version)
}
versionsMap[version] = struct{}{}
progress.Incr()
}
progress.Wait()
fmt.Println("the following versions of protoc will be used:", util.SetToSlice(versionsMap))
return nil
}
// StepInstallPlugins is used to install plugins
func StepInstallPlugins(ctx context.Context,
pluginManager pluginmanager.PluginManager,
configItems []configs.ConfigItem,
) error {
deduplicate := map[string]struct{}{}
for _, config := range configItems {
for _, pkg := range config.Config().Plugins {
deduplicate[pkg] = struct{}{}
}
}
progress := progressbar.GetProgressBar(len(deduplicate))
progress.SetPrefix("Install plugins")
pluginsMap := map[string]struct{}{}
for pkg := range deduplicate {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("invalid format: %s, should in path@version format", pkg)
}
if version == "latest" {
progress.SetSuffix("query latest version of %s", path)
data, err := pluginManager.GetPluginLatestVersion(ctx, path)
if err != nil {
return err
}
version = data
pkg = util.JoinGoPackageVersion(path, version)
}
progress.SetSuffix("check cache of %s", pkg)
exists, _, err := pluginManager.IsPluginInstalled(ctx, path, version)
if err != nil {
return err
}
if exists {
progress.SetSuffix("%s is cached", pkg)
} else {
progress.SetSuffix("installing %s", pkg)
_, err := pluginManager.InstallPlugin(ctx, path, version)
if err != nil {
panic(err)
}
progress.SetSuffix("%s installed", pkg)
}
pluginsMap[pkg] = struct{}{}
progress.Incr()
}
progress.SetSuffix("all plugins have been installed")
progress.Wait()
fmt.Println("the following plugins will be used:")
for pkg := range pluginsMap {
fmt.Printf(" %s\r\n", pkg)
}
return nil
}
// StepCompile is used to compile proto files
func StepCompile(ctx context.Context,
compilerManager compilermanager.CompilerManager,
targets []string,
) error {
progress := progressbar.GetProgressBar(len(targets))
progress.SetPrefix("Compile Proto Files")
c := concurrent.NewErrGroup(ctx, 10)
for _, target := range targets {
func(target string) {
c.Go(func(ctx context.Context) error {
progress.SetSuffix(target)
comp, err := compilerManager.GetCompiler(ctx, target)
if err != nil {
return err
}
if err := comp.Compile(ctx, target); err != nil {
return err
}
progress.Incr()
return nil
})
}(target)
}
if err := c.Wait(); err != nil {
return err
}
progress.Wait()
return nil
}
// StepPostAction is used to execute post actions
func StepPostAction(ctx context.Context,
actionsManager actionmanager.ActionManager,
configItems []configs.ConfigItem) error {
progress := progressbar.GetProgressBar(len(configItems))
progress.SetPrefix("PostAction")
for _, cfg := range configItems {
progress.SetSuffix(cfg.Path())
if err := actionsManager.ExecutePostAction(ctx, cfg); err != nil {
return err
}
progress.Incr()
}
progress.Wait()
return nil
}
// StepPostShell is used to execute post shell
func StepPostShell(ctx context.Context,
actionsManager actionmanager.ActionManager,
configItems []configs.ConfigItem) error {
progress := progressbar.GetProgressBar(len(configItems))
progress.SetPrefix("PostShell")
for _, cfg := range configItems {
progress.SetSuffix(cfg.Path())
if err := actionsManager.ExecutePostShell(ctx, cfg); err != nil {
return err
}
progress.Incr()
}
progress.Wait()
return nil
}
// Compile is used to compile proto files
func Compile(ctx context.Context, targets []string) error {
log := logger.NewDefault("compile")
log.SetLogLevel(logger.LevelError)
configManager, err := configmanager.NewConfigManager(log)
if err != nil {
return err
}
pluginManager, err := pluginmanager.NewPluginManager(pluginmanager.NewConfig(), log)
if err != nil {
return err
}
compilerManager, err := compilermanager.NewCompilerManager(ctx, log, configManager, pluginManager)
if err != nil {
return err
}
actionManager, err := actionmanager.NewActionManager(log)
if err != nil {
return err
}
configItems, err := StepLookUpConfigs(ctx, targets, configManager)
if err != nil {
return err
}
if err := StepInstallProtoc(ctx, pluginManager, configItems); err != nil {
return err
}
if err := StepInstallPlugins(ctx, pluginManager, configItems); err != nil {
return err
}
if err := StepCompile(ctx, compilerManager, targets); err != nil {
return err
}
if !command.IsDisableAction(ctx) {
if err := StepPostAction(ctx, actionManager, configItems); err != nil {
return err
}
if err := StepPostShell(ctx, actionManager, configItems); err != nil {
return err
}
} else {
log.LogWarn(nil, "PostAction and PostShell is skipped. If you need to allow execution, please append '-a' to command flags to enable")
}
log.LogInfo(nil, "Good job! you are ready to go :)")
return nil
}

118
pkg/bootstraps/tidy.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstraps
import (
"context"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/component/configmanager"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/logger"
"github.com/storyicon/powerproto/pkg/util/progressbar"
)
// StepTidyConfig is used to tidy configs by proto file targets
func StepTidyConfig(ctx context.Context, targets []string) error {
log := logger.NewDefault("tidy")
log.SetLogLevel(logger.LevelError)
configManager, err := configmanager.NewConfigManager(log)
if err != nil {
return err
}
pluginManager, err := pluginmanager.NewPluginManager(pluginmanager.NewConfig(), log)
if err != nil {
return err
}
configPaths := map[string]struct{}{}
for _, target := range targets {
cfg, err := configManager.GetConfig(ctx, target)
if err != nil {
return err
}
configPaths[cfg.Path()] = struct{}{}
}
if len(configPaths) == 0 {
return nil
}
progress := progressbar.GetProgressBar(len(configPaths))
progress.SetPrefix("tidy configs")
for path := range configPaths {
progress.SetSuffix(path)
err := StepTidyConfigFile(ctx, pluginManager, progress, path)
if err != nil {
return err
}
progress.Incr()
}
progress.SetSuffix("success!")
progress.Wait()
return nil
}
// StepTidyConfig is used clean config
// It will amend the 'latest' version to the latest version number in 'vx.y.z' format
func StepTidyConfigFile(ctx context.Context,
pluginManager pluginmanager.PluginManager,
progress progressbar.ProgressBar,
configFilePath string,
) error {
progress.SetSuffix("load config: %s", configFilePath)
configItems, err := configs.LoadConfigs(configFilePath)
if err != nil {
return err
}
var cleanable bool
for _, item := range configItems {
if item.Protoc == "latest" {
progress.SetSuffix("query latest version of protoc")
version, err := pluginManager.GetProtocLatestVersion(ctx)
if err != nil {
return err
}
item.Protoc = version
cleanable = true
}
for name, pkg := range item.Plugins {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("invalid package format: %s, should be in path@version format", pkg)
}
if version == "latest" {
progress.SetSuffix("query latest version of %s", path)
version, err := pluginManager.GetPluginLatestVersion(ctx, path)
if err != nil {
return err
}
item.Plugins[name] = util.JoinGoPackageVersion(path, version)
cleanable = true
}
}
}
if cleanable {
progress.SetSuffix("save %s", configFilePath)
if err := configs.SaveConfigs(configFilePath, configItems...); err != nil {
return err
}
}
progress.SetSuffix("config file tidied: %s", configFilePath)
return nil
}

31
pkg/component/actionmanager/actions/actions.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// CommonOptions is common options of action
type CommonOptions struct {
ConfigFilePath string
}
// ActionFunc defines the common prototype of action func
type ActionFunc func(ctx context.Context,
log logger.Logger,
args []string, options *CommonOptions) error

64
pkg/component/actionmanager/actions/copy.go Normal file
View file

@ -0,0 +1,64 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionCopy is used to copy directory or file from src to dest
// Its args prototype is:
// args: (src string, dest string)
func ActionCopy(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 2 || util.ContainsEmpty(args...) {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
source = args[0]
destination = args[1]
path = filepath.Dir(options.ConfigFilePath)
absSource = filepath.Join(path, source)
absDestination = filepath.Join(path, destination)
)
if filepath.IsAbs(source) {
return errors.Errorf("absolute source %s is not allowed in action move", source)
}
if filepath.IsAbs(destination) {
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "copy",
"from": absSource,
"to": absDestination,
}, "DryRun")
return nil
}
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}
return nil
}

68
pkg/component/actionmanager/actions/move.go Normal file
View file

@ -0,0 +1,68 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionMove is used to move directory or file from src to dest
// Its args prototype is:
// args: (src string, dest string)
func ActionMove(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 2 || util.ContainsEmpty(args...) {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
source = args[0]
destination = args[1]
path = filepath.Dir(options.ConfigFilePath)
absSource = filepath.Join(path, source)
absDestination = filepath.Join(path, destination)
)
if filepath.IsAbs(source) {
return errors.Errorf("absolute source %s is not allowed in action move", source)
}
if filepath.IsAbs(destination) {
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "move",
"from": absSource,
"to": absDestination,
}, "DryRun")
return nil
}
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}
if err := os.RemoveAll(absSource); err != nil {
return err
}
return nil
}

51
pkg/component/actionmanager/actions/remove.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionRemove is used to delete directories or files
// Its args prototype is:
// args: (path ...string)
func ActionRemove(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
for _, arg := range args {
if filepath.IsAbs(arg) {
return errors.Errorf("absolute path %s is not allowed in action remove", arg)
}
path := filepath.Join(filepath.Dir(options.ConfigFilePath), arg)
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "remove",
"target": path,
}, "DryRun")
return nil
}
if err := os.RemoveAll(path); err != nil {
return err
}
}
return nil
}

80
pkg/component/actionmanager/actions/replace.go Normal file
View file

@ -0,0 +1,80 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"bytes"
"context"
"io/fs"
"io/ioutil"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionReplace is used to replace text in bulk
// Its args prototype is:
// args: (pattern string, from string, to string)
// pattern is used to match files
func ActionReplace(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 3 {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
pattern = args[0]
path = filepath.Dir(options.ConfigFilePath)
from = args[1]
to = args[2]
)
if pattern == "" || from == "" {
return errors.Errorf("pattern and from arguments in action replace can not be empty")
}
if filepath.IsAbs(pattern) {
return errors.Errorf("absolute path %s is not allowed in action replace", pattern)
}
pattern = filepath.Join(path, pattern)
return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
return nil
}
matched, err := util.MatchPath(pattern, path)
if err != nil {
panic(err)
}
if matched {
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "replace",
"file": path,
"from": from,
"to": to,
}, "DryRun")
return nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
return err
}
data = bytes.ReplaceAll(data, []byte(from), []byte(to))
return ioutil.WriteFile(path, data, fs.ModePerm)
}
return nil
})
}

42
pkg/component/actionmanager/errors.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actionmanager
import (
"fmt"
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrPostShell defines the post shell command error
type ErrPostShell struct {
Path string
*command.ErrCommandExec
}
// ErrPostAction defines the post action error
type ErrPostAction struct {
Path string
Name string
Arguments []string
Err error
}
// Error implements the standard error interface
func (err *ErrPostAction) Error() string {
return fmt.Sprintf("failed to execute action: %s, path: %s, arguments: %s, err: %s",
err.Name, err.Path, err.Arguments, err.Err,
)
}

100
pkg/component/actionmanager/manager.go Normal file
View file

@ -0,0 +1,100 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actionmanager
import (
"context"
"fmt"
"path/filepath"
"github.com/storyicon/powerproto/pkg/component/actionmanager/actions"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionManager is used to manage actions
type ActionManager interface {
// ExecutePostShell is used to execute post shell in config item
ExecutePostShell(ctx context.Context, config configs.ConfigItem) error
// ExecutePostAction is used to execute post action in config item
ExecutePostAction(ctx context.Context, config configs.ConfigItem) error
}
// BasicActionManager is a basic implement of ActionManager
type BasicActionManager struct {
logger.Logger
// map[string]ActionFunc
actions map[string]actions.ActionFunc
}
// NewActionManager is used to create action manager
func NewActionManager(log logger.Logger) (ActionManager, error) {
return NewBasicActionManager(log)
}
// NewBasicActionManager is used to create a BasicActionManager
func NewBasicActionManager(log logger.Logger) (*BasicActionManager, error) {
return &BasicActionManager{
Logger: log.NewLogger("actionmanager"),
actions: map[string]actions.ActionFunc{
"move": actions.ActionMove,
"replace": actions.ActionReplace,
"remove": actions.ActionRemove,
"copy": actions.ActionCopy,
},
}, nil
}
// ExecutePostShell is used to execute post shell in config item
func (m *BasicActionManager) ExecutePostShell(ctx context.Context, config configs.ConfigItem) error {
script := config.Config().PostShell
if script == "" {
return nil
}
dir := filepath.Dir(config.Path())
_, err := command.Execute(ctx, m.Logger, dir, "/bin/sh", []string{
"-c", script,
}, nil)
if err != nil {
return &ErrPostShell{
Path: config.Path(),
ErrCommandExec: err,
}
}
return nil
}
// ExecutePostAction is used to execute post action in config item
func (m *BasicActionManager) ExecutePostAction(ctx context.Context, config configs.ConfigItem) error {
for _, action := range config.Config().PostActions {
actionFunc, ok := m.actions[action.Name]
if !ok {
return fmt.Errorf("unknown action: %s", action.Name)
}
if err := actionFunc(ctx, m.Logger, action.Args, &actions.CommonOptions{
ConfigFilePath: config.Path(),
}); err != nil {
return &ErrPostAction{
Path: config.Path(),
Name: action.Name,
Arguments: action.Args,
Err: err,
}
}
}
return nil
}

179
pkg/component/compilermanager/compiler.go Normal file
View file

@ -0,0 +1,179 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"context"
"fmt"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// Compiler is used to compile proto file
type Compiler interface {
// Compile is used to compile proto file
Compile(ctx context.Context, protoFilePath string) error
// GetConfig is used to return config that the compiler used
GetConfig(ctx context.Context) configs.ConfigItem
}
var _ Compiler = &BasicCompiler{}
// BasicCompiler is the basic implement of Compiler
type BasicCompiler struct {
logger.Logger
config configs.ConfigItem
pluginManager pluginmanager.PluginManager
protocPath string
arguments []string
dir string
}
// NewCompiler is used to create a compiler
func NewCompiler(
ctx context.Context,
log logger.Logger,
pluginManager pluginmanager.PluginManager,
config configs.ConfigItem,
) (Compiler, error) {
return NewBasicCompiler(ctx, log, pluginManager, config)
}
// NewBasicCompiler is used to create a basic compiler
func NewBasicCompiler(
ctx context.Context,
log logger.Logger,
pluginManager pluginmanager.PluginManager,
config configs.ConfigItem,
) (*BasicCompiler, error) {
basic := &BasicCompiler{
Logger: log.NewLogger("compiler"),
config: config,
pluginManager: pluginManager,
}
if err := basic.calcProto(ctx); err != nil {
return nil, err
}
if err := basic.calcArguments(ctx); err != nil {
return nil, err
}
if err := basic.calcDir(ctx); err != nil {
return nil, err
}
return basic, nil
}
// Compile is used to compile proto file
func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error {
arguments := make([]string, len(b.arguments))
copy(arguments, b.arguments)
arguments = append(arguments, protoFilePath)
_, err := command.Execute(ctx,
b.Logger, b.dir, b.protocPath, arguments, nil)
if err != nil {
return &ErrCompile{
ErrCommandExec: err,
}
}
return nil
}
// GetConfig is used to return config that the compiler used
func (b *BasicCompiler) GetConfig(ctx context.Context) configs.ConfigItem {
return b.config
}
func (b *BasicCompiler) calcDir(ctx context.Context) error {
if dir := b.config.Config().ProtocWorkDir; dir != "" {
dir = util.RenderPathWithEnv(dir)
if !filepath.IsAbs(dir) {
dir = filepath.Join(b.config.Path(), dir)
}
b.dir = dir
} else {
b.dir = filepath.Dir(b.config.Path())
}
return nil
}
func (b *BasicCompiler) calcProto(ctx context.Context) error {
cfg := b.config
protocVersion := cfg.Config().Protoc
if protocVersion == "latest" {
latestVersion, err := b.pluginManager.GetProtocLatestVersion(ctx)
if err != nil {
return err
}
protocVersion = latestVersion
}
localPath, err := b.pluginManager.InstallProtoc(ctx, protocVersion)
if err != nil {
return err
}
b.protocPath = localPath
return nil
}
func (b *BasicCompiler) calcArguments(ctx context.Context) error {
cfg := b.config
arguments := make([]string, len(cfg.Config().Options))
copy(arguments, cfg.Config().Options)
dir := filepath.Dir(cfg.Path())
// build import paths
for _, path := range cfg.Config().ImportPaths {
if path == consts.KeyPowerProtoInclude {
path = b.pluginManager.IncludePath(ctx)
}
path = util.RenderPathWithEnv(path)
if !filepath.IsAbs(path) {
path = filepath.Join(dir, path)
}
arguments = append(arguments, "--proto_path="+path)
}
// build plugin options
for name, pkg := range cfg.Config().Plugins {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("failed to parse: %s", pkg)
}
if version == "latest" {
latestVersion, err := b.pluginManager.GetPluginLatestVersion(ctx, path)
if err != nil {
return err
}
version = latestVersion
}
local, err := b.pluginManager.InstallPlugin(ctx, path, version)
if err != nil {
return errors.Wrap(err, "failed to get plugin path")
}
arg := fmt.Sprintf("--plugin=%s=%s", name, local)
arguments = append(arguments, arg)
}
b.arguments = arguments
return nil
}

24
pkg/component/compilermanager/errors.go Normal file
View file

@ -0,0 +1,24 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrCompile defines the compile error
type ErrCompile struct {
*command.ErrCommandExec
}

88
pkg/component/compilermanager/manager.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"context"
"sync"
"github.com/storyicon/powerproto/pkg/component/configmanager"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// CompilerManager is to manage compiler
type CompilerManager interface {
// GetCompiler is used to get compiler of specified proto file path
GetCompiler(ctx context.Context, protoFilePath string) (Compiler, error)
}
// BasicCompilerManager is the basic implement of CompilerManager
type BasicCompilerManager struct {
logger.Logger
configManager configmanager.ConfigManager
pluginManager pluginmanager.PluginManager
tree map[string]Compiler
treeLock sync.RWMutex
}
var _ CompilerManager = &BasicCompilerManager{}
// NewCompilerManager is used to create CompilerManager
func NewCompilerManager(ctx context.Context,
log logger.Logger,
configManager configmanager.ConfigManager,
pluginManager pluginmanager.PluginManager,
) (CompilerManager, error) {
return NewBasicCompilerManager(ctx, log, configManager, pluginManager)
}
// BasicCompilerManager is used to create basic CompilerManager
func NewBasicCompilerManager(ctx context.Context,
log logger.Logger,
configManager configmanager.ConfigManager,
pluginManager pluginmanager.PluginManager,
) (*BasicCompilerManager, error) {
return &BasicCompilerManager{
Logger: log.NewLogger("compilermanager"),
configManager: configManager,
pluginManager: pluginManager,
tree: map[string]Compiler{},
}, nil
}
// GetCompiler is used to get compiler of specified proto file path
func (b *BasicCompilerManager) GetCompiler(ctx context.Context, protoFilePath string) (Compiler, error) {
config, err := b.configManager.GetConfig(ctx, protoFilePath)
if err != nil {
return nil, err
}
b.treeLock.Lock()
defer b.treeLock.Unlock()
c, ok := b.tree[config.Path()]
if ok {
return c, nil
}
compiler, err := NewCompiler(ctx, b.Logger, b.pluginManager, config)
if err != nil {
return nil, err
}
b.tree[config.ID()] = compiler
return compiler, nil
}

103
pkg/component/configmanager/manager.go Normal file
View file

@ -0,0 +1,103 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package configmanager
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ConfigManager is used to manage config
type ConfigManager interface {
// GetCompiler is used to get config of specified proto file path
GetConfig(ctx context.Context, protoFilePath string) (configs.ConfigItem, error)
}
// NewConfigManager is used to create ConfigManager
func NewConfigManager(log logger.Logger) (ConfigManager, error) {
return NewBasicConfigManager(log)
}
// BasicConfigManager is the basic implement of ConfigManager
type BasicConfigManager struct {
logger.Logger
tree map[string][]configs.ConfigItem
treeLock sync.RWMutex
}
// New is used to create a basic ConfigManager
func NewBasicConfigManager(log logger.Logger) (*BasicConfigManager, error) {
return &BasicConfigManager{
Logger: log.NewLogger("configmanager"),
tree: map[string][]configs.ConfigItem{},
}, nil
}
// GetCompiler is used to get config of specified proto file path
func (b *BasicConfigManager) GetConfig(ctx context.Context, protoFilePath string) (configs.ConfigItem, error) {
possiblePath := configs.ListConfigPaths(filepath.Dir(protoFilePath))
for _, configFilePath := range possiblePath {
items, err := b.loadConfig(configFilePath)
if err != nil {
return nil, err
}
dir := filepath.Dir(configFilePath)
for _, config := range items {
for _, scope := range config.Config().Scopes {
scopePath := filepath.Join(dir, scope)
if strings.Contains(protoFilePath, scopePath) {
return config, nil
}
}
}
}
return nil, errors.Errorf("unable to find config: %s", protoFilePath)
}
func (b *BasicConfigManager) loadConfig(configFilePath string) ([]configs.ConfigItem, error) {
b.treeLock.Lock()
defer b.treeLock.Unlock()
configItems, ok := b.tree[configFilePath]
if !ok {
fileInfo, err := os.Stat(configFilePath)
if err != nil {
if os.IsNotExist(err) {
b.tree[configFilePath] = nil
return nil, nil
}
return nil, err
}
if fileInfo.IsDir() {
b.tree[configFilePath] = nil
return nil, nil
}
data, err := configs.LoadConfigItems(configFilePath)
if err != nil {
return nil, errors.WithMessagef(err, "failed to decode: %s", configFilePath)
}
b.tree[configFilePath] = data
return data, nil
}
return configItems, nil
}

43
pkg/component/pluginmanager/errors.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"fmt"
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrGoInstall defines the go install error
type ErrGoInstall struct {
*command.ErrCommandExec
}
// ErrGoList defines the go list error
type ErrGoList struct {
*command.ErrCommandExec
}
// ErrHTTPDownload defines the download error
type ErrHTTPDownload struct {
Url string
Err error
Code int
}
// Error implements the error interface
func (err *ErrHTTPDownload) Error() string {
return fmt.Sprintf("failed to download %s, code: %d, err: %s", err.Url, err.Code, err.Err)
}

215
pkg/component/pluginmanager/manager.go Normal file
View file

@ -0,0 +1,215 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"errors"
"io/fs"
"os"
"strings"
"sync"
"time"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/logger"
)
var defaultExecuteTimeout = time.Second * 60
// PluginManager is used to manage plugins
type PluginManager interface {
// GetPluginLatestVersion is used to get the latest version of plugin
GetPluginLatestVersion(ctx context.Context, path string) (string, error)
// ListPluginVersions is used to list the versions of plugin
ListPluginVersions(ctx context.Context, path string) ([]string, error)
// IsPluginInstalled is used to check whether the plugin is installed
IsPluginInstalled(ctx context.Context, path string, version string) (bool, string, error)
// InstallPlugin is used to install plugin
InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
// GetProtocLatestVersion is used to geet the latest version of protoc
GetProtocLatestVersion(ctx context.Context) (string, error)
// ListProtocVersions is used to list protoc version
ListProtocVersions(ctx context.Context) ([]string, error)
// IsProtocInstalled is used to check whether the protoc is installed
IsProtocInstalled(ctx context.Context, version string) (bool, string, error)
// InstallProtoc is used to install protoc of specified version
InstallProtoc(ctx context.Context, version string) (local string, err error)
// IncludePath returns the default include path
IncludePath(ctx context.Context) string
}
// Config defines the config of PluginManager
type Config struct {
StorageDir string `json:"storage"`
}
// NewConfig is used to create config
func NewConfig() *Config {
return &Config{
StorageDir: consts.GetHomeDir(),
}
}
// NewPluginManager is used to create PluginManager
func NewPluginManager(cfg *Config, log logger.Logger) (PluginManager, error) {
return NewBasicPluginManager(cfg.StorageDir, log)
}
// BasicPluginManager is the basic implement of PluginManager
type BasicPluginManager struct {
logger.Logger
storageDir string
versions map[string][]string
versionsLock sync.RWMutex
}
// NewBasicPluginManager is used to create basic PluginManager
func NewBasicPluginManager(storageDir string, log logger.Logger) (*BasicPluginManager, error) {
return &BasicPluginManager{
Logger: log.NewLogger("pluginmanager"),
storageDir: storageDir,
versions: map[string][]string{},
}, nil
}
// GetPluginLatestVersion is used to get the latest version of plugin
func (b *BasicPluginManager) GetPluginLatestVersion(ctx context.Context, path string) (string, error) {
versions, err := b.ListPluginVersions(ctx, path)
if err != nil {
return "", err
}
if len(versions) == 0 {
return "", errors.New("no version list")
}
return versions[len(versions)-1], nil
}
// ListPluginVersions is used to list the versions of plugin
func (b *BasicPluginManager) ListPluginVersions(ctx context.Context, path string) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions[path]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListsGoPackageVersionsAmbiguously(ctx, b.Logger, path)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions[path] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsPluginInstalled is used to check whether the plugin is installed
func (b *BasicPluginManager) IsPluginInstalled(ctx context.Context, path string, version string) (bool, string, error) {
return IsPluginInstalled(ctx, b.storageDir, path, version)
}
// InstallPlugin is used to install plugin
func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, version string) (local string, err error) {
return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
}
// GetProtocLatestVersion is used to geet the latest version of protoc
func (b *BasicPluginManager) GetProtocLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListProtocVersions(ctx)
if err != nil {
return "", err
}
if len(versions) == 0 {
return "", errors.New("no version list")
}
return versions[len(versions)-1], nil
}
// ListProtocVersions is used to list protoc version
func (b *BasicPluginManager) ListProtocVersions(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions["protoc"]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListGitTags(ctx, b.Logger, consts.ProtobufRepository)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions["protoc"] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsProtocInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
return IsProtocInstalled(ctx, b.storageDir, version)
}
// InstallProtoc is used to install protoc of specified version
func (b *BasicPluginManager) InstallProtoc(ctx context.Context, version string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
local := PathForProtoc(b.storageDir, version)
exists, err := util.IsFileExists(local)
if err != nil {
return "", err
}
if exists {
return local, nil
}
release, err := GetProtocRelease(ctx, version)
if err != nil {
return "", err
}
defer release.Clear()
// merge include files
includeDir := PathForInclude(b.storageDir)
if err := util.CopyDirectory(release.GetIncludePath(), includeDir); err != nil {
return "", err
}
// download protoc file
if err := util.CopyFile(release.GetProtocPath(), local); err != nil {
return "", err
}
// * it is required on unix system
if err := os.Chmod(local, fs.ModePerm); err != nil {
return "", err
}
return local, nil
}
// IncludePath returns the default include path
func (b *BasicPluginManager) IncludePath(ctx context.Context) string {
return PathForInclude(b.storageDir)
}

92
pkg/component/pluginmanager/paths.go Normal file
View file

@ -0,0 +1,92 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"path"
"path/filepath"
"golang.org/x/mod/module"
"github.com/storyicon/powerproto/pkg/util"
)
// PathForInclude is used to get the local directory of include files
func PathForInclude(storageDir string) string {
return filepath.Join(storageDir, "include")
}
// PathForProtoc is used to get the local binary location where the specified version protoc should be stored
func PathForProtoc(storageDir string, version string) string {
return filepath.Join(storageDir, "protoc", version, util.GetBinaryFileName("protoc"))
}
// GetPluginPath is used to get the plugin path
func GetPluginPath(path string, version string) (string, error) {
enc, err := module.EscapePath(path)
if err != nil {
return "", err
}
encVer, err := module.EscapeVersion(version)
if err != nil {
return "", err
}
return filepath.Join(enc + "@" + encVer), nil
}
// PathForPluginDir is used to get the local directory where the specified version plug-in should be stored
func PathForPluginDir(storageDir string, path string, version string) (string, error) {
pluginPath, err := GetPluginPath(path, version)
if err != nil {
return "", err
}
return filepath.Join(storageDir, "plugins", pluginPath), nil
}
// PathForPlugin is used to get the binary path of plugin
func PathForPlugin(storageDir string, path string, version string) (string, error) {
name := GetGoPkgExecName(path)
dir, err := PathForPluginDir(storageDir, path, version)
if err != nil {
return "", err
}
return filepath.Join(dir, util.GetBinaryFileName(name)), nil
}
// isVersionElement reports whether s is a well-formed path version element:
// v2, v3, v10, etc, but not v0, v05, v1.
// `src\cmd\go\internal\load\pkg.go:1209`
func isVersionElement(s string) bool {
if len(s) < 2 || s[0] != 'v' || s[1] == '0' || s[1] == '1' && len(s) == 2 {
return false
}
for i := 1; i < len(s); i++ {
if s[i] < '0' || '9' < s[i] {
return false
}
}
return true
}
// GetGoPkgExecName is used to parse binary name from pkg uri
// `src\cmd\go\internal\load\pkg.go:1595`
func GetGoPkgExecName(pkgPath string) string {
_, elem := path.Split(pkgPath)
if elem != pkgPath && isVersionElement(elem) {
_, elem = path.Split(path.Dir(pkgPath))
}
return elem
}

174
pkg/component/pluginmanager/plugin.go Normal file
View file

@ -0,0 +1,174 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-multierror"
jsoniter "github.com/json-iterator/go"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// IsPluginInstalled is used to check whether a plugin is installed
func IsPluginInstalled(ctx context.Context,
storageDir string,
path string, version string) (bool, string, error) {
local, err := PathForPlugin(storageDir, path, version)
if err != nil {
return false, "", err
}
exists, err := util.IsFileExists(local)
if err != nil {
return false, "", err
}
if exists {
return true, local, nil
}
return false, "", nil
}
// InstallPluginUsingGo is used to install plugin using golang
func InstallPluginUsingGo(ctx context.Context,
log logger.Logger,
storageDir string,
path string, version string) (string, error) {
exists, local, err := IsPluginInstalled(ctx, storageDir, path, version)
if err != nil {
return "", err
}
if exists {
return local, nil
}
local, err = PathForPlugin(storageDir, path, version)
if err != nil {
return "", err
}
dir := filepath.Dir(local)
uri := util.JoinGoPackageVersion(path, version)
_, err2 := command.Execute(ctx, log, "", "go", []string{
"install", uri,
}, []string{"GOBIN=" + dir, "GO111MODULE=on"})
if err2 != nil {
return "", &ErrGoInstall{
ErrCommandExec: err2,
}
}
return local, nil
}
// ///////////////// Version Control /////////////////
// Module defines the model of go list data
type Module struct {
Path string // module path
Version string // module version
Versions []string // available module versions (with -versions)
Replace *Module // replaced by this module
Time *time.Time // time version was created
Update *Module // available update, if any (with -u)
Main bool // is this the main module?
Indirect bool // is this module only an indirect dependency of main module?
Dir string // directory holding files for this module, if any
GoMod string // path to go.mod file used when loading this module, if any
GoVersion string // go version used in module
Retracted string // retraction information, if any (with -retracted or -u)
Error *ModuleError // error loading module
}
// ModuleError defines the module error
type ModuleError struct {
Err string // the error itself
}
// ListGoPackageVersions is list go package versions
func ListGoPackageVersions(ctx context.Context, log logger.Logger, path string) ([]string, error) {
// query from latest version
// If latest is not specified here, the queried version
// may be restricted to the current project go.mod/go.sum
pkg := util.JoinGoPackageVersion(path, "latest")
data, err := command.Execute(ctx, log, "", "go", []string{
"list", "-m", "-json", "-versions", pkg,
}, []string{
"GO111MODULE=on",
})
if err != nil {
return nil, &ErrGoList{
ErrCommandExec: err,
}
}
var module Module
if err := jsoniter.Unmarshal(data, &module); err != nil {
return nil, err
}
if len(module.Versions) != 0 {
return module.Versions, nil
}
return []string{module.Version}, nil
}
// ListsGoPackageVersionsAmbiguously is used to list go package versions ambiguously
func ListsGoPackageVersionsAmbiguously(ctx context.Context, log logger.Logger, pkg string) ([]string, error) {
type Result struct {
err error
pkg string
versions []string
}
items := strings.Split(pkg, "/")
dataMap := make([]*Result, len(items))
notify := make(chan struct{}, 1)
maxIndex := len(items) - 1
for i := maxIndex; i >= 1; i-- {
go func(i int) {
pkg := strings.Join(items[0:i+1], "/")
versions, err := ListGoPackageVersions(context.TODO(), log, pkg)
dataMap[maxIndex-i] = &Result{
pkg: pkg,
versions: versions,
err: err,
}
notify <- struct{}{}
}(i)
}
OutLoop:
for {
select {
case <-notify:
var errs error
for _, data := range dataMap {
if data == nil {
continue OutLoop
}
if data.err != nil {
errs = multierror.Append(errs, data.err)
}
if data.versions != nil {
return data.versions, nil
}
}
return nil, errs
case <-ctx.Done():
return nil, ctx.Err()
}
}
}

27
pkg/component/pluginmanager/pluginmanager_suite_test.go Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPluginmanager(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pluginmanager Suite")
}

73
pkg/component/pluginmanager/pluginmanager_test.go Normal file
View file

@ -0,0 +1,73 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager_test
import (
"context"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/util/logger"
)
var _ = Describe("Pluginmanager", func() {
cfg := pluginmanager.NewConfig()
cfg.StorageDir, _ = filepath.Abs("./tests")
const pluginPkg = "google.golang.org/protobuf/cmd/protoc-gen-go"
var manager pluginmanager.PluginManager
It("should able to init", func() {
pluginManager, err := pluginmanager.NewPluginManager(cfg, logger.NewDefault("pluginmanager"))
Expect(err).To(BeNil())
Expect(pluginManager).To(Not(BeNil()))
manager = pluginManager
})
It("should able to install protoc", func() {
versions, err := manager.ListProtocVersions(context.TODO())
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetProtocLatestVersion(context.TODO())
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallProtoc(context.TODO(), latestVersion)
Expect(err).To(BeNil())
exists, local, err := manager.IsProtocInstalled(context.TODO(), latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
It("should able to install plugin", func() {
versions, err := manager.ListPluginVersions(context.TODO(), pluginPkg)
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetPluginLatestVersion(context.TODO(), pluginPkg)
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallPlugin(context.TODO(), pluginPkg, latestVersion)
Expect(err).To(BeNil())
Expect(len(local) > 0).To(BeTrue())
exists, local, err := manager.IsPluginInstalled(context.TODO(), pluginPkg, latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
})

186
pkg/component/pluginmanager/protoc.go Normal file
View file

@ -0,0 +1,186 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/mholt/archiver"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ProtocRelease defines the release of protoc
type ProtocRelease struct {
workspace string
}
// GetIncludePath is used to get the include path
func (p *ProtocRelease) GetIncludePath() string {
return filepath.Join(p.workspace, "include")
}
// GetProtocPath is used to get the protoc path
func (p *ProtocRelease) GetProtocPath() string {
return filepath.Join(p.workspace, "bin", util.GetBinaryFileName("protoc"))
}
// Clear is used to clear the workspace
func (p *ProtocRelease) Clear() error {
return os.RemoveAll(p.workspace)
}
// GetProtocRelease is used to download protoc release
func GetProtocRelease(ctx context.Context, version string) (*ProtocRelease, error) {
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
suffix, err := inferProtocReleaseSuffix()
if err != nil {
return nil, err
}
filename := fmt.Sprintf("protoc-%s-%s.zip", version, suffix)
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/"+
"releases/download/v%s/%s", version, filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &ProtocRelease{
workspace: workspace,
}, nil
}
// IsProtocInstalled is used to check whether the protoc version is installed
func IsProtocInstalled(ctx context.Context, storageDir string, version string) (bool, string, error) {
local := PathForProtoc(storageDir, version)
exists, err := util.IsFileExists(local)
if err != nil {
return false, "", err
}
return exists, local, nil
}
// ErrGitList defines the git list error
type ErrGitList struct {
*command.ErrCommandExec
}
// ListGitTags is used to list the git tags of specified repository
func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
}, nil)
if err != nil {
return nil, &ErrGitList{
ErrCommandExec: err,
}
}
var tags []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
if strings.HasPrefix(f[1], "refs/tags/") {
tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
}
}
return tags, nil
}
func inferProtocReleaseSuffix() (string, error) {
goos := strings.ToLower(runtime.GOOS)
arch := strings.ToLower(runtime.GOARCH)
switch goos {
case "linux":
switch arch {
case "arm64":
return "linux-aarch_64", nil
case "ppc64le":
return "linux-ppcle_64", nil
case "s390x":
return "linux-s390_64", nil
case "386":
return "linux-x86_32", nil
case "amd64":
return "linux-x86_64", nil
}
case "darwin":
return "osx-x86_64", nil
case "windows":
switch arch {
case "386":
return "win32", nil
case "amd64":
return "win64", nil
}
}
return "", errors.New("protoc did not release on this platform")
}
func downloadFile(resp *http.Response, destination string) error {
if err := os.MkdirAll(filepath.Dir(destination), fs.ModePerm); err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.Errorf("unexpected code %d for url: %s", resp.StatusCode, resp.Request.URL.String())
}
file, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fs.ModePerm)
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
return nil
}

109
pkg/configs/configs.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package configs
import (
"bytes"
"io/fs"
"io/ioutil"
"path/filepath"
"gopkg.in/yaml.v3"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
)
// Config defines the config model
type Config struct {
Scopes []string `json:"scopes" yaml:"scopes"`
Protoc string `json:"protoc" yaml:"protoc"`
ProtocWorkDir string `json:"protocWorkDir" yaml:"protocWorkDir"`
Plugins map[string]string `json:"plugins" yaml:"plugins"`
Options []string `json:"options" yaml:"options"`
ImportPaths []string `json:"importPaths" yaml:"importPaths"`
PostActions []*PostAction `json:"postActions" yaml:"postActions"`
PostShell string `json:"postShell" yaml:"postShell"`
}
// PostAction defines the Action model
type PostAction struct {
Name string `json:"name" yaml:"name"`
Args []string `json:"args" yaml:"args"`
}
// SaveConfigs is used to save configs into files
func SaveConfigs(path string, configs ...*Config) error {
parts := make([][]byte, 0, len(configs))
for _, config := range configs {
data, err := yaml.Marshal(config)
if err != nil {
return err
}
parts = append(parts, data)
}
data := bytes.Join(parts, []byte("\r\n---\r\n"))
if err := ioutil.WriteFile(path, data, fs.ModePerm); err != nil {
return err
}
return nil
}
// LoadConfigs is used to load config from specified path
func LoadConfigs(path string) ([]*Config, error) {
raw, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
items, err := util.SplitYAML(raw)
if err != nil {
return nil, err
}
var ret []*Config
for _, item := range items {
var config Config
if err := yaml.Unmarshal(item, &config); err != nil {
return nil, err
}
ret = append(ret, &config)
}
return ret, nil
}
// Similar to LoadConfigs, but obtains the abstraction of the Config prototype structure
func LoadConfigItems(path string) ([]ConfigItem, error) {
data, err := LoadConfigs(path)
if err != nil {
return nil, err
}
return GetConfigItems(data, path), nil
}
// ListConfigPaths is used to list all possible config paths
func ListConfigPaths(sourceDir string) []string {
var paths []string
cur := sourceDir
for {
filePath := filepath.Join(cur, consts.ConfigFileName)
paths = append(paths, filePath)
next := filepath.Dir(cur)
if next == cur {
break
}
cur = next
}
paths = append(paths, consts.PathForGlobalConfig())
return paths
}

67
pkg/configs/item.go Normal file
View file

@ -0,0 +1,67 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package configs
import (
"fmt"
)
// ConfigItem is the wrapper of Config
type ConfigItem interface {
// ID is used to return to config unique id
ID() string
// Path is used to return the config path
Path() string
// Config is used to return the Config
Config() *Config
}
// GetConfigs is used to generate Config from given config entity
func GetConfigItems(data []*Config, path string) []ConfigItem {
ret := make([]ConfigItem, 0, len(data))
for i, item := range data {
ret = append(ret, newConfigItem(item, path, i))
}
return ret
}
type configItem struct {
id string
c *Config
path string
}
// ID is used to return to config unique id
func (c *configItem) ID() string {
return c.id
}
// Path is used to return the config path
func (c *configItem) Path() string {
return c.path
}
// Config is used to return the Config
func (c *configItem) Config() *Config {
return c.c
}
func newConfigItem(c *Config, path string, idx int) ConfigItem {
return &configItem{
id: fmt.Sprintf("%s:%d", path, idx),
c: c,
path: path,
}
}

69
pkg/consts/consts.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package consts
import (
"os"
"path/filepath"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// defines a set of const value
const (
// ConfigFileName defines the config file name
ConfigFileName = "powerproto.yaml"
// The default include can be referenced by this key in import paths
KeyPowerProtoInclude = "$POWERPROTO_INCLUDE"
// Defines the program directory of PowerProto, including various binary and include files
EnvHomeDir = "POWERPROTO_HOME"
// ProtobufRepository defines the protobuf repository
ProtobufRepository = "https://github.com/protocolbuffers/protobuf"
)
var homeDir string
var log = logger.NewDefault("consts")
// GetHomeDir is used to get cached homeDir
func GetHomeDir() string {
return homeDir
}
// PathForGlobalConfig is used to get path of global config
func PathForGlobalConfig() string {
return filepath.Join(GetHomeDir(), ConfigFileName)
}
func getHomeDir() (string, error) {
val := os.Getenv(EnvHomeDir)
if val != "" {
return filepath.Abs(val)
}
homeDir, err := os.UserHomeDir()
if err != nil {
return "", err
}
return filepath.Join(homeDir, ".powerproto"), nil
}
func init() {
var err error
homeDir, err = getHomeDir()
if err != nil {
log.LogFatal(nil, "Please set the working directory of PowerProto by "+
"configuring the environment variable %s", EnvHomeDir)
}
}

109
pkg/util/command/command.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package command
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/logger"
)
type dryRun struct{}
type ignoreDryRun struct{}
type disableAction struct{}
// WithIgnoreDryRun is used to inject ignore dryRun flag into context
func WithIgnoreDryRun(ctx context.Context) context.Context {
return context.WithValue(ctx, ignoreDryRun{}, "true")
}
// WithDisableAction is used to disable post action/shell
func WithDisableAction(ctx context.Context) context.Context {
return context.WithValue(ctx, disableAction{}, "true")
}
// WithDryRun is used to inject dryRun flag into context
func WithDryRun(ctx context.Context) context.Context {
return context.WithValue(ctx, dryRun{}, "true")
}
// IsDisableAction is used to decide whether to disable PostAction and PostShell
func IsDisableAction(ctx context.Context) bool {
return ctx.Value(disableAction{}) != nil
}
// IsIgnoreDryRun is used to determine whether it is currently in ignore DryRun mode
func IsIgnoreDryRun(ctx context.Context) bool {
return ctx.Value(ignoreDryRun{}) != nil
}
// IsDryRun is used to determine whether it is currently in DryRun mode
func IsDryRun(ctx context.Context) bool {
return ctx.Value(dryRun{}) != nil
}
// Execute is used to execute commands, return stdout and execute errors
func Execute(ctx context.Context,
log logger.Logger,
dir string, name string, arguments []string, env []string) ([]byte, *ErrCommandExec) {
cmd := exec.CommandContext(ctx, name, arguments...)
cmd.Env = append(os.Environ(), env...)
cmd.Dir = dir
if IsDryRun(ctx) && !IsIgnoreDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"command": cmd.String(),
"dir": cmd.Dir,
}, "DryRun")
return nil, nil
}
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, &ErrCommandExec{
Err: err,
Dir: cmd.Dir,
Command: cmd.String(),
ExitCode: util.GetExitCode(err),
Stdout: stdout.String(),
Stderr: stderr.String(),
}
}
return stdout.Bytes(), nil
}
// ErrCommandExec defines the command error
type ErrCommandExec struct {
Err error
Dir string
Command string
ExitCode int
Stdout string
Stderr string
}
// Error implements the error interface
func (err *ErrCommandExec) Error() string {
return fmt.Sprintf("failed to execute %s in %s, stderr: %s, exit code %d, %s",
err.Command, err.Dir, err.Stderr, err.ExitCode, err.Err,
)
}

27
pkg/util/command/command_suite_test.go Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package command_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestCommand(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Command Suite")
}

38
pkg/util/command/command_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package command_test
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/storyicon/powerproto/pkg/util/command"
)
var _ = Describe("Command", func() {
ctx := context.Background()
It("should able to dryRun", func() {
Expect(command.IsDryRun(ctx)).To(BeFalse())
ctx = command.WithDryRun(ctx)
Expect(command.IsDryRun(ctx)).To(BeTrue())
})
It("should able to ignore dryRun", func() {
Expect(command.IsIgnoreDryRun(ctx)).To(BeFalse())
ctx = command.WithIgnoreDryRun(ctx)
Expect(command.IsIgnoreDryRun(ctx)).To(BeTrue())
})
})

38
pkg/util/concurrent/buffer.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package concurrent
import (
"io"
"sync"
)
// Buffer is goroutine safety buffer
type Buffer struct {
writer io.Writer
mutex sync.Mutex
}
// Write implements the io.Writer
func (s *Buffer) Write(p []byte) (n int, err error) {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.writer.Write(p)
}
// NewBuffer is used to create a new buffer
func NewBuffer(writer io.Writer) io.Writer {
return &Buffer{writer: writer}
}

72
pkg/util/concurrent/errgroup.go Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package concurrent
import (
"context"
"sync"
)
// ErrGroup is another ErrGroup implement
type ErrGroup struct {
ctx context.Context
cancel context.CancelFunc
limit chan struct{}
errOnce sync.Once
err error
wg sync.WaitGroup
}
// NewErrGroup is used to create a new ErrGroup
func NewErrGroup(ctx context.Context, concurrency int) *ErrGroup {
ctx, cancel := context.WithCancel(ctx)
return &ErrGroup{
ctx: ctx,
cancel: cancel,
limit: make(chan struct{}, concurrency),
}
}
// Wait is used to wait ErrGroup finish
func (g *ErrGroup) Wait() error {
g.wg.Wait()
g.cancel()
return g.err
}
// Go is used to start a new goroutine
func (g *ErrGroup) Go(f func(ctx context.Context) error) {
if g.err != nil {
return
}
g.limit <- struct{}{}
g.wg.Add(1)
go func() {
defer func() {
<-g.limit
g.wg.Done()
}()
if err := f(g.ctx); err != nil {
g.cancel()
g.errOnce.Do(func() {
g.err = err
g.cancel()
})
}
}()
}

131
pkg/util/file.go Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"io"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"github.com/bmatcuk/doublestar"
filecopy "github.com/otiai10/copy"
"github.com/pkg/errors"
)
// MatchPath is used to match path with specified pattern
func MatchPath(pattern string, path string) (bool, error) {
return doublestar.PathMatch(pattern, path)
}
// CopyDirectory is used to copy directory
// If dst already exists, it will be merged
func CopyDirectory(src, dst string) error {
return filecopy.Copy(src, dst)
}
// CopyFile is used to copy file from src to dst
func CopyFile(src, dst string) error {
sourceFileStat, err := os.Stat(src)
if err != nil {
return err
}
if !sourceFileStat.Mode().IsRegular() {
return errors.Errorf("%s is not a regular file", src)
}
source, err := os.Open(src)
if err != nil {
return err
}
defer source.Close()
if err := os.MkdirAll(filepath.Dir(dst), fs.ModePerm); err != nil {
return err
}
destination, err := os.Create(dst)
if err != nil {
return err
}
defer destination.Close()
_, err = io.Copy(destination, source)
return err
}
// IsFileExists is used to check whether the file exists
func IsFileExists(path string) (bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
if info.IsDir() {
return false, errors.Errorf("%s is not a file", path)
}
return true, nil
}
// IsDirExists is used to check whether the dir exists
func IsDirExists(path string) (bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
if !info.IsDir() {
return false, errors.Errorf("%s is not a directory", path)
}
return true, nil
}
// GetFilesWithExtRecursively is used to recursively list files with a specific suffix
// expectExt should contain the prefix '.'
func GetFilesWithExtRecursively(target string, targetExt string) ([]string, error) {
var data []string
err := filepath.Walk(target, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext == targetExt {
data = append(data, path)
}
return nil
})
return data, err
}
// GetFilesWithExtRecursively is used to list files with a specific suffix
// expectExt should contain the prefix '.'
func GetFilesWithExt(dir string, targetExt string) ([]string, error) {
children, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
var data []string
for _, child := range children {
if child.IsDir() {
continue
}
if ext := filepath.Ext(child.Name()); ext != targetExt {
continue
}
data = append(data, filepath.Join(dir, child.Name()))
}
return data, nil
}

217
pkg/util/logger/logger.go Normal file
View file

@ -0,0 +1,217 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logger
import (
"fmt"
"io"
"os"
"sort"
"strings"
"github.com/fatih/color"
jsoniter "github.com/json-iterator/go"
"github.com/prometheus/client_golang/prometheus"
)
// Logger defines the basic log library implementation
type Logger interface {
// LogDebug print a message with debug level.
LogDebug(fields map[string]interface{}, format string, args ...interface{})
// LogInfo print a message with info level.
LogInfo(fields map[string]interface{}, format string, args ...interface{})
// LogWarn print a message with warn level.
LogWarn(fields map[string]interface{}, format string, args ...interface{})
// LogError print a message with error level.
LogError(fields map[string]interface{}, format string, args ...interface{})
// LogFatal print a message with fatal level.
LogFatal(fields map[string]interface{}, format string, args ...interface{})
// NewLogger is used to derive a new child Logger
NewLogger(component string) Logger
// SetLogLevel is used to set log level
SetLogLevel(level Level) Logger
}
// BasicLogger simply implements Logger
type BasicLogger struct {
cfg *Config
component string
registerer prometheus.Registerer
}
// Config defines the config structure
type Config struct {
Pretty bool
Level Level
}
type Level struct {
Name string
Color color.Attribute
Index int
Writer io.Writer
}
var (
LevelDebug = Level{Name: "debug", Color: color.FgWhite, Index: 0, Writer: os.Stdout}
LevelInfo = Level{Name: "info", Color: color.FgWhite, Index: 1, Writer: os.Stdout}
LevelWarn = Level{Name: "warn", Color: color.FgYellow, Index: 2, Writer: os.Stderr}
LevelError = Level{Name: "error", Color: color.FgHiRed, Index: 3, Writer: os.Stderr}
LevelFatal = Level{Name: "fatal", Color: color.FgRed, Index: 4, Writer: os.Stderr}
)
// NewConfig is used to init config with default values
func NewConfig() *Config {
return &Config{
Pretty: false,
Level: LevelDebug,
}
}
// NewDefault is used to initialize a simple Logger
func NewDefault(component string) Logger {
logger, err := New(NewConfig(), component, prometheus.DefaultRegisterer)
if err != nil {
panic(err)
}
return logger
}
// New is used to init service
func New(cfg *Config, component string, registerer prometheus.Registerer) (Logger, error) {
if cfg == nil {
cfg = NewConfig()
}
service := &BasicLogger{
cfg: cfg,
component: component,
registerer: registerer,
}
return service, nil
}
// LogDebug print a message with debug level.
func (b *BasicLogger) LogDebug(fields map[string]interface{}, format string, args ...interface{}) {
b.log(LevelDebug, fields, format, args...)
}
// LogInfo print a message with info level.
func (b *BasicLogger) LogInfo(fields map[string]interface{}, format string, args ...interface{}) {
b.log(LevelInfo, fields, format, args...)
}
// LogWarn print a message with warn level.
func (b *BasicLogger) LogWarn(fields map[string]interface{}, format string, args ...interface{}) {
b.log(LevelWarn, fields, format, args...)
}
// LogError print a message with error level.
func (b *BasicLogger) LogError(fields map[string]interface{}, format string, args ...interface{}) {
b.log(LevelError, fields, format, args...)
}
// LogFatal print a message with fatal level.
func (b *BasicLogger) LogFatal(fields map[string]interface{}, format string, args ...interface{}) {
b.log(LevelFatal, fields, format, args...)
}
// NewLogger is used to derive a new child Logger
func (b *BasicLogger) NewLogger(component string) Logger {
name := strings.Join([]string{b.component, component}, ".")
logger, err := New(b.cfg, name, b.registerer)
if err != nil {
b.LogWarn(map[string]interface{}{
"name": name,
}, "failed to extend logger: %s", err)
return b
}
return logger
}
// SetLogLevel is used to set log level
func (b *BasicLogger) SetLogLevel(level Level) Logger {
b.cfg.Level = level
return b
}
func (b *BasicLogger) log(level Level, fields map[string]interface{}, format string, args ...interface{}) {
if b.cfg.Level.Index > level.Index {
return
}
if fields == nil {
fields = map[string]interface{}{}
}
// if b.cfg.Level == LevelDebug {
// if _, file, line, ok := runtime.Caller(4); ok {
// fields["file"] = fmt.Sprintf("%s:%d", path.Base(file), line)
// }
// if b.component != "" {
// fields["component"] = b.component
// }
// }
if b.cfg.Pretty {
dict := map[string]interface{}{}
for key, val := range fields {
dict[key] = val
}
dict["level"] = level.Name
dict["message"] = fmt.Sprintf(format, args...)
_ = jsoniter.NewEncoder(level.Writer).Encode(dict)
return
}
var buf []byte
buf = appendString(buf, fmt.Sprintf(format, args...))
if len(fields) > 0 {
buf = appendTab(buf)
buf = appendFields(buf, fields)
}
buf = appendLF(buf)
fmt.Fprintf(level.Writer, color.New(level.Color).Sprint(string(buf)))
if level == LevelFatal {
os.Exit(1)
}
}
func appendFields(dst []byte, fields map[string]interface{}) []byte {
keys := make([]string, 0, len(fields))
for key := range fields {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
dst = appendField(dst, key, fields[key])
dst = appendTab(dst)
}
return dst
}
func appendTab(dst []byte) []byte {
return append(dst, ' ')
}
func appendField(dst []byte, key string, val interface{}) []byte {
dst = appendString(dst, fmt.Sprintf("%s=%v", key, val))
return dst
}
func appendString(dst []byte, s string) []byte {
dst = append(dst, []byte(s)...)
return dst
}
func appendLF(dst []byte) []byte {
return append(dst, '\n')
}

123
pkg/util/progressbar/progressbar.go Normal file
View file

@ -0,0 +1,123 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package progressbar
import (
"fmt"
"github.com/fatih/color"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
)
// ProgressBar implements a customizable progress bar
type ProgressBar interface {
// Incr is used to increase progress
Incr()
// Wait is used to wait for the rendering of the progress bar to complete
Wait()
// SetPrefix is used to set the prefix of progress bar
SetPrefix(format string, args ...interface{})
// SetSuffix is used to set the suffix of progress bar
SetSuffix(format string, args ...interface{})
}
type progressBar struct {
container *mpb.Progress
bar *mpb.Bar
prefix string
suffix string
}
// SetPrefix is used to set the prefix of progress bar
func (s *progressBar) SetPrefix(format string, args ...interface{}) {
s.prefix = fmt.Sprintf(format, args...)
}
// SetSuffix is used to set the suffix of progress bar
func (s *progressBar) SetSuffix(format string, args ...interface{}) {
s.suffix = fmt.Sprintf(format, args...)
}
func newEmbedProgressBar(container *mpb.Progress, bar *mpb.Bar) *progressBar {
return &progressBar{
container: container,
bar: bar,
}
}
func (s *progressBar) Incr() {
s.bar.Increment()
}
func (s *progressBar) Wait() {
s.container.Wait()
}
func getSpinner() []string {
activeState := "[ " + color.GreenString("●") + " ] "
defaultState := "[ ] "
return []string{
activeState,
activeState,
activeState,
defaultState,
defaultState,
defaultState,
}
}
// GetProgressBar is used to get progress bar
func GetProgressBar(count int) ProgressBar {
var progressBar *progressBar
container := mpb.New()
bar := container.Add(int64(count),
mpb.NewBarFiller(mpb.BarStyle().Lbound("[").
Filler(color.GreenString("=")).
Tip(color.GreenString(">")).Padding(" ").Rbound("]")),
mpb.PrependDecorators(
func() decor.Decorator {
frames := getSpinner()
var count uint
return decor.Any(func(statistics decor.Statistics) string {
if statistics.Completed {
return frames[0]
}
frame := frames[count%uint(len(frames))]
count++
return frame
})
}(),
decor.Any(func(statistics decor.Statistics) string {
if progressBar != nil {
return progressBar.prefix
}
return ""
}),
),
mpb.AppendDecorators(
decor.NewPercentage("%d "),
decor.Any(func(statistics decor.Statistics) string {
if progressBar != nil {
return fmt.Sprintf("(%d/%d) %s", statistics.Current, count, progressBar.suffix)
}
return ""
}),
),
mpb.BarWidth(15),
)
progressBar = newEmbedProgressBar(container, bar)
return progressBar
}

106
pkg/util/util.go Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"syscall"
)
// ContainsEmpty is used to check whether items contains empty string
func ContainsEmpty(items ...string) bool {
for _, item := range items {
if item == "" {
return true
}
}
return false
}
// SetToSlice is used to convert set<string> to slice<string>
func SetToSlice(set map[string]struct{}) []string {
data := make([]string, 0, len(set))
for key := range set {
data = append(data, key)
}
return data
}
// GetMapKeys is used to get the keys of map
func GetMapKeys(dict map[string]string) []string {
data := make([]string, 0, len(dict))
for key := range dict {
data = append(data, key)
}
return data
}
// GetExitCode is used to parse exit code from cmd error
func GetExitCode(err error) int {
if exitErr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
// This works on both Unix and Windows. Although package
// syscall is generally platform dependent, WaitStatus is
// defined for both Unix and Windows and in both cases has
// an ExitStatus() method with the same signature.
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
return status.ExitStatus()
}
}
return 1
}
var regexpEnvironmentVar = regexp.MustCompile(`\$[A-Za-z_]+`)
// RenderPathWithEnv is used to render path with environment
func RenderPathWithEnv(path string) string {
matches := regexpEnvironmentVar.FindAllString(path, -1)
for _, match := range matches {
path = strings.ReplaceAll(path, match, os.Getenv(match[1:]))
}
return filepath.Clean(path)
}
// SplitGoPackageVersion is used to split go package version
func SplitGoPackageVersion(pkg string) (path string, version string, ok bool) {
i := strings.Index(pkg, "@")
if i == -1 {
return "", "", false
}
return pkg[:i], pkg[i+1:], true
}
// JoinGoPackageVersion is used to join go path and versions
func JoinGoPackageVersion(path, version string) string {
return strings.Join([]string{
path, version,
}, "@")
}
// GetBinaryFileName is used to get os based binary file name
func GetBinaryFileName(name string) string {
if runtime.GOOS == "windows" {
if !strings.HasSuffix(name, ".exe") {
return name + ".exe"
}
return name
}
return name
}

74
pkg/util/yaml.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"github.com/hashicorp/go-multierror"
"gopkg.in/yaml.v2"
)
// SplitYAML is used to split yaml
func SplitYAML(data []byte) ([][]byte, error) {
decoder := yaml.NewDecoder(bytes.NewReader(data))
var parts [][]byte
for {
var value interface{}
err := decoder.Decode(&value)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
part, err := yaml.Marshal(value)
if err != nil {
return nil, err
}
parts = append(parts, part)
}
return parts, nil
}
// DumpYaml is used to dump yaml into stdout
func DumpYaml(cfg interface{}) {
out, err := yaml.Marshal(cfg)
if err != nil {
fmt.Fprintln(os.Stderr, err)
} else {
fmt.Printf("%s\n", out)
}
}
// LoadConfig read YAML-formatted config from filename into cfg.
func LoadConfig(filename string, pointer interface{}) error {
buf, err := ioutil.ReadFile(filename)
if err != nil {
return multierror.Prefix(err, "Error reading config file")
}
err = yaml.UnmarshalStrict(buf, pointer)
if err != nil {
return multierror.Prefix(err, "Error parsing config file")
}
return nil
}