Signed-off-by: storyicon <yuanchao@bilibili.com>
This commit is contained in:
storyicon 2021-07-21 00:24:43 +08:00
commit 9aac714c32
No known key found for this signature in database
GPG key ID: 245915D985F966CF
47 changed files with 5480 additions and 0 deletions

View file

@ -0,0 +1,31 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// CommonOptions is common options of action
type CommonOptions struct {
ConfigFilePath string
}
// ActionFunc defines the common prototype of action func
type ActionFunc func(ctx context.Context,
log logger.Logger,
args []string, options *CommonOptions) error

View file

@ -0,0 +1,64 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionCopy is used to copy directory or file from src to dest
// Its args prototype is:
// args: (src string, dest string)
func ActionCopy(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 2 || util.ContainsEmpty(args...) {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
source = args[0]
destination = args[1]
path = filepath.Dir(options.ConfigFilePath)
absSource = filepath.Join(path, source)
absDestination = filepath.Join(path, destination)
)
if filepath.IsAbs(source) {
return errors.Errorf("absolute source %s is not allowed in action move", source)
}
if filepath.IsAbs(destination) {
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "copy",
"from": absSource,
"to": absDestination,
}, "DryRun")
return nil
}
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}
return nil
}

View file

@ -0,0 +1,68 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionMove is used to move directory or file from src to dest
// Its args prototype is:
// args: (src string, dest string)
func ActionMove(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 2 || util.ContainsEmpty(args...) {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
source = args[0]
destination = args[1]
path = filepath.Dir(options.ConfigFilePath)
absSource = filepath.Join(path, source)
absDestination = filepath.Join(path, destination)
)
if filepath.IsAbs(source) {
return errors.Errorf("absolute source %s is not allowed in action move", source)
}
if filepath.IsAbs(destination) {
return errors.Errorf("absolute destination %s is not allowed in action move", destination)
}
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "move",
"from": absSource,
"to": absDestination,
}, "DryRun")
return nil
}
if err := util.CopyDirectory(absSource, absDestination); err != nil {
return err
}
if err := os.RemoveAll(absSource); err != nil {
return err
}
return nil
}

View file

@ -0,0 +1,51 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"context"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionRemove is used to delete directories or files
// Its args prototype is:
// args: (path ...string)
func ActionRemove(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
for _, arg := range args {
if filepath.IsAbs(arg) {
return errors.Errorf("absolute path %s is not allowed in action remove", arg)
}
path := filepath.Join(filepath.Dir(options.ConfigFilePath), arg)
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "remove",
"target": path,
}, "DryRun")
return nil
}
if err := os.RemoveAll(path); err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,80 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actions
import (
"bytes"
"context"
"io/fs"
"io/ioutil"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionReplace is used to replace text in bulk
// Its args prototype is:
// args: (pattern string, from string, to string)
// pattern is used to match files
func ActionReplace(ctx context.Context, log logger.Logger, args []string, options *CommonOptions) error {
if len(args) != 3 {
return errors.Errorf("expected length of args is 3, but received %d", len(args))
}
var (
pattern = args[0]
path = filepath.Dir(options.ConfigFilePath)
from = args[1]
to = args[2]
)
if pattern == "" || from == "" {
return errors.Errorf("pattern and from arguments in action replace can not be empty")
}
if filepath.IsAbs(pattern) {
return errors.Errorf("absolute path %s is not allowed in action replace", pattern)
}
pattern = filepath.Join(path, pattern)
return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
return nil
}
matched, err := util.MatchPath(pattern, path)
if err != nil {
panic(err)
}
if matched {
if command.IsDryRun(ctx) {
log.LogInfo(map[string]interface{}{
"action": "replace",
"file": path,
"from": from,
"to": to,
}, "DryRun")
return nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
return err
}
data = bytes.ReplaceAll(data, []byte(from), []byte(to))
return ioutil.WriteFile(path, data, fs.ModePerm)
}
return nil
})
}

View file

@ -0,0 +1,42 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actionmanager
import (
"fmt"
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrPostShell defines the post shell command error
type ErrPostShell struct {
Path string
*command.ErrCommandExec
}
// ErrPostAction defines the post action error
type ErrPostAction struct {
Path string
Name string
Arguments []string
Err error
}
// Error implements the standard error interface
func (err *ErrPostAction) Error() string {
return fmt.Sprintf("failed to execute action: %s, path: %s, arguments: %s, err: %s",
err.Name, err.Path, err.Arguments, err.Err,
)
}

View file

@ -0,0 +1,100 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package actionmanager
import (
"context"
"fmt"
"path/filepath"
"github.com/storyicon/powerproto/pkg/component/actionmanager/actions"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ActionManager is used to manage actions
type ActionManager interface {
// ExecutePostShell is used to execute post shell in config item
ExecutePostShell(ctx context.Context, config configs.ConfigItem) error
// ExecutePostAction is used to execute post action in config item
ExecutePostAction(ctx context.Context, config configs.ConfigItem) error
}
// BasicActionManager is a basic implement of ActionManager
type BasicActionManager struct {
logger.Logger
// map[string]ActionFunc
actions map[string]actions.ActionFunc
}
// NewActionManager is used to create action manager
func NewActionManager(log logger.Logger) (ActionManager, error) {
return NewBasicActionManager(log)
}
// NewBasicActionManager is used to create a BasicActionManager
func NewBasicActionManager(log logger.Logger) (*BasicActionManager, error) {
return &BasicActionManager{
Logger: log.NewLogger("actionmanager"),
actions: map[string]actions.ActionFunc{
"move": actions.ActionMove,
"replace": actions.ActionReplace,
"remove": actions.ActionRemove,
"copy": actions.ActionCopy,
},
}, nil
}
// ExecutePostShell is used to execute post shell in config item
func (m *BasicActionManager) ExecutePostShell(ctx context.Context, config configs.ConfigItem) error {
script := config.Config().PostShell
if script == "" {
return nil
}
dir := filepath.Dir(config.Path())
_, err := command.Execute(ctx, m.Logger, dir, "/bin/sh", []string{
"-c", script,
}, nil)
if err != nil {
return &ErrPostShell{
Path: config.Path(),
ErrCommandExec: err,
}
}
return nil
}
// ExecutePostAction is used to execute post action in config item
func (m *BasicActionManager) ExecutePostAction(ctx context.Context, config configs.ConfigItem) error {
for _, action := range config.Config().PostActions {
actionFunc, ok := m.actions[action.Name]
if !ok {
return fmt.Errorf("unknown action: %s", action.Name)
}
if err := actionFunc(ctx, m.Logger, action.Args, &actions.CommonOptions{
ConfigFilePath: config.Path(),
}); err != nil {
return &ErrPostAction{
Path: config.Path(),
Name: action.Name,
Arguments: action.Args,
Err: err,
}
}
}
return nil
}

View file

@ -0,0 +1,179 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"context"
"fmt"
"path/filepath"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// Compiler is used to compile proto file
type Compiler interface {
// Compile is used to compile proto file
Compile(ctx context.Context, protoFilePath string) error
// GetConfig is used to return config that the compiler used
GetConfig(ctx context.Context) configs.ConfigItem
}
var _ Compiler = &BasicCompiler{}
// BasicCompiler is the basic implement of Compiler
type BasicCompiler struct {
logger.Logger
config configs.ConfigItem
pluginManager pluginmanager.PluginManager
protocPath string
arguments []string
dir string
}
// NewCompiler is used to create a compiler
func NewCompiler(
ctx context.Context,
log logger.Logger,
pluginManager pluginmanager.PluginManager,
config configs.ConfigItem,
) (Compiler, error) {
return NewBasicCompiler(ctx, log, pluginManager, config)
}
// NewBasicCompiler is used to create a basic compiler
func NewBasicCompiler(
ctx context.Context,
log logger.Logger,
pluginManager pluginmanager.PluginManager,
config configs.ConfigItem,
) (*BasicCompiler, error) {
basic := &BasicCompiler{
Logger: log.NewLogger("compiler"),
config: config,
pluginManager: pluginManager,
}
if err := basic.calcProto(ctx); err != nil {
return nil, err
}
if err := basic.calcArguments(ctx); err != nil {
return nil, err
}
if err := basic.calcDir(ctx); err != nil {
return nil, err
}
return basic, nil
}
// Compile is used to compile proto file
func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error {
arguments := make([]string, len(b.arguments))
copy(arguments, b.arguments)
arguments = append(arguments, protoFilePath)
_, err := command.Execute(ctx,
b.Logger, b.dir, b.protocPath, arguments, nil)
if err != nil {
return &ErrCompile{
ErrCommandExec: err,
}
}
return nil
}
// GetConfig is used to return config that the compiler used
func (b *BasicCompiler) GetConfig(ctx context.Context) configs.ConfigItem {
return b.config
}
func (b *BasicCompiler) calcDir(ctx context.Context) error {
if dir := b.config.Config().ProtocWorkDir; dir != "" {
dir = util.RenderPathWithEnv(dir)
if !filepath.IsAbs(dir) {
dir = filepath.Join(b.config.Path(), dir)
}
b.dir = dir
} else {
b.dir = filepath.Dir(b.config.Path())
}
return nil
}
func (b *BasicCompiler) calcProto(ctx context.Context) error {
cfg := b.config
protocVersion := cfg.Config().Protoc
if protocVersion == "latest" {
latestVersion, err := b.pluginManager.GetProtocLatestVersion(ctx)
if err != nil {
return err
}
protocVersion = latestVersion
}
localPath, err := b.pluginManager.InstallProtoc(ctx, protocVersion)
if err != nil {
return err
}
b.protocPath = localPath
return nil
}
func (b *BasicCompiler) calcArguments(ctx context.Context) error {
cfg := b.config
arguments := make([]string, len(cfg.Config().Options))
copy(arguments, cfg.Config().Options)
dir := filepath.Dir(cfg.Path())
// build import paths
for _, path := range cfg.Config().ImportPaths {
if path == consts.KeyPowerProtoInclude {
path = b.pluginManager.IncludePath(ctx)
}
path = util.RenderPathWithEnv(path)
if !filepath.IsAbs(path) {
path = filepath.Join(dir, path)
}
arguments = append(arguments, "--proto_path="+path)
}
// build plugin options
for name, pkg := range cfg.Config().Plugins {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("failed to parse: %s", pkg)
}
if version == "latest" {
latestVersion, err := b.pluginManager.GetPluginLatestVersion(ctx, path)
if err != nil {
return err
}
version = latestVersion
}
local, err := b.pluginManager.InstallPlugin(ctx, path, version)
if err != nil {
return errors.Wrap(err, "failed to get plugin path")
}
arg := fmt.Sprintf("--plugin=%s=%s", name, local)
arguments = append(arguments, arg)
}
b.arguments = arguments
return nil
}

View file

@ -0,0 +1,24 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrCompile defines the compile error
type ErrCompile struct {
*command.ErrCommandExec
}

View file

@ -0,0 +1,88 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compilermanager
import (
"context"
"sync"
"github.com/storyicon/powerproto/pkg/component/configmanager"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// CompilerManager is to manage compiler
type CompilerManager interface {
// GetCompiler is used to get compiler of specified proto file path
GetCompiler(ctx context.Context, protoFilePath string) (Compiler, error)
}
// BasicCompilerManager is the basic implement of CompilerManager
type BasicCompilerManager struct {
logger.Logger
configManager configmanager.ConfigManager
pluginManager pluginmanager.PluginManager
tree map[string]Compiler
treeLock sync.RWMutex
}
var _ CompilerManager = &BasicCompilerManager{}
// NewCompilerManager is used to create CompilerManager
func NewCompilerManager(ctx context.Context,
log logger.Logger,
configManager configmanager.ConfigManager,
pluginManager pluginmanager.PluginManager,
) (CompilerManager, error) {
return NewBasicCompilerManager(ctx, log, configManager, pluginManager)
}
// BasicCompilerManager is used to create basic CompilerManager
func NewBasicCompilerManager(ctx context.Context,
log logger.Logger,
configManager configmanager.ConfigManager,
pluginManager pluginmanager.PluginManager,
) (*BasicCompilerManager, error) {
return &BasicCompilerManager{
Logger: log.NewLogger("compilermanager"),
configManager: configManager,
pluginManager: pluginManager,
tree: map[string]Compiler{},
}, nil
}
// GetCompiler is used to get compiler of specified proto file path
func (b *BasicCompilerManager) GetCompiler(ctx context.Context, protoFilePath string) (Compiler, error) {
config, err := b.configManager.GetConfig(ctx, protoFilePath)
if err != nil {
return nil, err
}
b.treeLock.Lock()
defer b.treeLock.Unlock()
c, ok := b.tree[config.Path()]
if ok {
return c, nil
}
compiler, err := NewCompiler(ctx, b.Logger, b.pluginManager, config)
if err != nil {
return nil, err
}
b.tree[config.ID()] = compiler
return compiler, nil
}

View file

@ -0,0 +1,103 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package configmanager
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/configs"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ConfigManager is used to manage config
type ConfigManager interface {
// GetCompiler is used to get config of specified proto file path
GetConfig(ctx context.Context, protoFilePath string) (configs.ConfigItem, error)
}
// NewConfigManager is used to create ConfigManager
func NewConfigManager(log logger.Logger) (ConfigManager, error) {
return NewBasicConfigManager(log)
}
// BasicConfigManager is the basic implement of ConfigManager
type BasicConfigManager struct {
logger.Logger
tree map[string][]configs.ConfigItem
treeLock sync.RWMutex
}
// New is used to create a basic ConfigManager
func NewBasicConfigManager(log logger.Logger) (*BasicConfigManager, error) {
return &BasicConfigManager{
Logger: log.NewLogger("configmanager"),
tree: map[string][]configs.ConfigItem{},
}, nil
}
// GetCompiler is used to get config of specified proto file path
func (b *BasicConfigManager) GetConfig(ctx context.Context, protoFilePath string) (configs.ConfigItem, error) {
possiblePath := configs.ListConfigPaths(filepath.Dir(protoFilePath))
for _, configFilePath := range possiblePath {
items, err := b.loadConfig(configFilePath)
if err != nil {
return nil, err
}
dir := filepath.Dir(configFilePath)
for _, config := range items {
for _, scope := range config.Config().Scopes {
scopePath := filepath.Join(dir, scope)
if strings.Contains(protoFilePath, scopePath) {
return config, nil
}
}
}
}
return nil, errors.Errorf("unable to find config: %s", protoFilePath)
}
func (b *BasicConfigManager) loadConfig(configFilePath string) ([]configs.ConfigItem, error) {
b.treeLock.Lock()
defer b.treeLock.Unlock()
configItems, ok := b.tree[configFilePath]
if !ok {
fileInfo, err := os.Stat(configFilePath)
if err != nil {
if os.IsNotExist(err) {
b.tree[configFilePath] = nil
return nil, nil
}
return nil, err
}
if fileInfo.IsDir() {
b.tree[configFilePath] = nil
return nil, nil
}
data, err := configs.LoadConfigItems(configFilePath)
if err != nil {
return nil, errors.WithMessagef(err, "failed to decode: %s", configFilePath)
}
b.tree[configFilePath] = data
return data, nil
}
return configItems, nil
}

View file

@ -0,0 +1,43 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"fmt"
"github.com/storyicon/powerproto/pkg/util/command"
)
// ErrGoInstall defines the go install error
type ErrGoInstall struct {
*command.ErrCommandExec
}
// ErrGoList defines the go list error
type ErrGoList struct {
*command.ErrCommandExec
}
// ErrHTTPDownload defines the download error
type ErrHTTPDownload struct {
Url string
Err error
Code int
}
// Error implements the error interface
func (err *ErrHTTPDownload) Error() string {
return fmt.Sprintf("failed to download %s, code: %d, err: %s", err.Url, err.Code, err.Err)
}

View file

@ -0,0 +1,215 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"errors"
"io/fs"
"os"
"strings"
"sync"
"time"
"github.com/storyicon/powerproto/pkg/consts"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/logger"
)
var defaultExecuteTimeout = time.Second * 60
// PluginManager is used to manage plugins
type PluginManager interface {
// GetPluginLatestVersion is used to get the latest version of plugin
GetPluginLatestVersion(ctx context.Context, path string) (string, error)
// ListPluginVersions is used to list the versions of plugin
ListPluginVersions(ctx context.Context, path string) ([]string, error)
// IsPluginInstalled is used to check whether the plugin is installed
IsPluginInstalled(ctx context.Context, path string, version string) (bool, string, error)
// InstallPlugin is used to install plugin
InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
// GetProtocLatestVersion is used to geet the latest version of protoc
GetProtocLatestVersion(ctx context.Context) (string, error)
// ListProtocVersions is used to list protoc version
ListProtocVersions(ctx context.Context) ([]string, error)
// IsProtocInstalled is used to check whether the protoc is installed
IsProtocInstalled(ctx context.Context, version string) (bool, string, error)
// InstallProtoc is used to install protoc of specified version
InstallProtoc(ctx context.Context, version string) (local string, err error)
// IncludePath returns the default include path
IncludePath(ctx context.Context) string
}
// Config defines the config of PluginManager
type Config struct {
StorageDir string `json:"storage"`
}
// NewConfig is used to create config
func NewConfig() *Config {
return &Config{
StorageDir: consts.GetHomeDir(),
}
}
// NewPluginManager is used to create PluginManager
func NewPluginManager(cfg *Config, log logger.Logger) (PluginManager, error) {
return NewBasicPluginManager(cfg.StorageDir, log)
}
// BasicPluginManager is the basic implement of PluginManager
type BasicPluginManager struct {
logger.Logger
storageDir string
versions map[string][]string
versionsLock sync.RWMutex
}
// NewBasicPluginManager is used to create basic PluginManager
func NewBasicPluginManager(storageDir string, log logger.Logger) (*BasicPluginManager, error) {
return &BasicPluginManager{
Logger: log.NewLogger("pluginmanager"),
storageDir: storageDir,
versions: map[string][]string{},
}, nil
}
// GetPluginLatestVersion is used to get the latest version of plugin
func (b *BasicPluginManager) GetPluginLatestVersion(ctx context.Context, path string) (string, error) {
versions, err := b.ListPluginVersions(ctx, path)
if err != nil {
return "", err
}
if len(versions) == 0 {
return "", errors.New("no version list")
}
return versions[len(versions)-1], nil
}
// ListPluginVersions is used to list the versions of plugin
func (b *BasicPluginManager) ListPluginVersions(ctx context.Context, path string) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions[path]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListsGoPackageVersionsAmbiguously(ctx, b.Logger, path)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions[path] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsPluginInstalled is used to check whether the plugin is installed
func (b *BasicPluginManager) IsPluginInstalled(ctx context.Context, path string, version string) (bool, string, error) {
return IsPluginInstalled(ctx, b.storageDir, path, version)
}
// InstallPlugin is used to install plugin
func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, version string) (local string, err error) {
return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
}
// GetProtocLatestVersion is used to geet the latest version of protoc
func (b *BasicPluginManager) GetProtocLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListProtocVersions(ctx)
if err != nil {
return "", err
}
if len(versions) == 0 {
return "", errors.New("no version list")
}
return versions[len(versions)-1], nil
}
// ListProtocVersions is used to list protoc version
func (b *BasicPluginManager) ListProtocVersions(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions["protoc"]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListGitTags(ctx, b.Logger, consts.ProtobufRepository)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions["protoc"] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsProtocInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsProtocInstalled(ctx context.Context, version string) (bool, string, error) {
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
return IsProtocInstalled(ctx, b.storageDir, version)
}
// InstallProtoc is used to install protoc of specified version
func (b *BasicPluginManager) InstallProtoc(ctx context.Context, version string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
local := PathForProtoc(b.storageDir, version)
exists, err := util.IsFileExists(local)
if err != nil {
return "", err
}
if exists {
return local, nil
}
release, err := GetProtocRelease(ctx, version)
if err != nil {
return "", err
}
defer release.Clear()
// merge include files
includeDir := PathForInclude(b.storageDir)
if err := util.CopyDirectory(release.GetIncludePath(), includeDir); err != nil {
return "", err
}
// download protoc file
if err := util.CopyFile(release.GetProtocPath(), local); err != nil {
return "", err
}
// * it is required on unix system
if err := os.Chmod(local, fs.ModePerm); err != nil {
return "", err
}
return local, nil
}
// IncludePath returns the default include path
func (b *BasicPluginManager) IncludePath(ctx context.Context) string {
return PathForInclude(b.storageDir)
}

View file

@ -0,0 +1,92 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"path"
"path/filepath"
"golang.org/x/mod/module"
"github.com/storyicon/powerproto/pkg/util"
)
// PathForInclude is used to get the local directory of include files
func PathForInclude(storageDir string) string {
return filepath.Join(storageDir, "include")
}
// PathForProtoc is used to get the local binary location where the specified version protoc should be stored
func PathForProtoc(storageDir string, version string) string {
return filepath.Join(storageDir, "protoc", version, util.GetBinaryFileName("protoc"))
}
// GetPluginPath is used to get the plugin path
func GetPluginPath(path string, version string) (string, error) {
enc, err := module.EscapePath(path)
if err != nil {
return "", err
}
encVer, err := module.EscapeVersion(version)
if err != nil {
return "", err
}
return filepath.Join(enc + "@" + encVer), nil
}
// PathForPluginDir is used to get the local directory where the specified version plug-in should be stored
func PathForPluginDir(storageDir string, path string, version string) (string, error) {
pluginPath, err := GetPluginPath(path, version)
if err != nil {
return "", err
}
return filepath.Join(storageDir, "plugins", pluginPath), nil
}
// PathForPlugin is used to get the binary path of plugin
func PathForPlugin(storageDir string, path string, version string) (string, error) {
name := GetGoPkgExecName(path)
dir, err := PathForPluginDir(storageDir, path, version)
if err != nil {
return "", err
}
return filepath.Join(dir, util.GetBinaryFileName(name)), nil
}
// isVersionElement reports whether s is a well-formed path version element:
// v2, v3, v10, etc, but not v0, v05, v1.
// `src\cmd\go\internal\load\pkg.go:1209`
func isVersionElement(s string) bool {
if len(s) < 2 || s[0] != 'v' || s[1] == '0' || s[1] == '1' && len(s) == 2 {
return false
}
for i := 1; i < len(s); i++ {
if s[i] < '0' || '9' < s[i] {
return false
}
}
return true
}
// GetGoPkgExecName is used to parse binary name from pkg uri
// `src\cmd\go\internal\load\pkg.go:1595`
func GetGoPkgExecName(pkgPath string) string {
_, elem := path.Split(pkgPath)
if elem != pkgPath && isVersionElement(elem) {
_, elem = path.Split(path.Dir(pkgPath))
}
return elem
}

View file

@ -0,0 +1,174 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-multierror"
jsoniter "github.com/json-iterator/go"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// IsPluginInstalled is used to check whether a plugin is installed
func IsPluginInstalled(ctx context.Context,
storageDir string,
path string, version string) (bool, string, error) {
local, err := PathForPlugin(storageDir, path, version)
if err != nil {
return false, "", err
}
exists, err := util.IsFileExists(local)
if err != nil {
return false, "", err
}
if exists {
return true, local, nil
}
return false, "", nil
}
// InstallPluginUsingGo is used to install plugin using golang
func InstallPluginUsingGo(ctx context.Context,
log logger.Logger,
storageDir string,
path string, version string) (string, error) {
exists, local, err := IsPluginInstalled(ctx, storageDir, path, version)
if err != nil {
return "", err
}
if exists {
return local, nil
}
local, err = PathForPlugin(storageDir, path, version)
if err != nil {
return "", err
}
dir := filepath.Dir(local)
uri := util.JoinGoPackageVersion(path, version)
_, err2 := command.Execute(ctx, log, "", "go", []string{
"install", uri,
}, []string{"GOBIN=" + dir, "GO111MODULE=on"})
if err2 != nil {
return "", &ErrGoInstall{
ErrCommandExec: err2,
}
}
return local, nil
}
// ///////////////// Version Control /////////////////
// Module defines the model of go list data
type Module struct {
Path string // module path
Version string // module version
Versions []string // available module versions (with -versions)
Replace *Module // replaced by this module
Time *time.Time // time version was created
Update *Module // available update, if any (with -u)
Main bool // is this the main module?
Indirect bool // is this module only an indirect dependency of main module?
Dir string // directory holding files for this module, if any
GoMod string // path to go.mod file used when loading this module, if any
GoVersion string // go version used in module
Retracted string // retraction information, if any (with -retracted or -u)
Error *ModuleError // error loading module
}
// ModuleError defines the module error
type ModuleError struct {
Err string // the error itself
}
// ListGoPackageVersions is list go package versions
func ListGoPackageVersions(ctx context.Context, log logger.Logger, path string) ([]string, error) {
// query from latest version
// If latest is not specified here, the queried version
// may be restricted to the current project go.mod/go.sum
pkg := util.JoinGoPackageVersion(path, "latest")
data, err := command.Execute(ctx, log, "", "go", []string{
"list", "-m", "-json", "-versions", pkg,
}, []string{
"GO111MODULE=on",
})
if err != nil {
return nil, &ErrGoList{
ErrCommandExec: err,
}
}
var module Module
if err := jsoniter.Unmarshal(data, &module); err != nil {
return nil, err
}
if len(module.Versions) != 0 {
return module.Versions, nil
}
return []string{module.Version}, nil
}
// ListsGoPackageVersionsAmbiguously is used to list go package versions ambiguously
func ListsGoPackageVersionsAmbiguously(ctx context.Context, log logger.Logger, pkg string) ([]string, error) {
type Result struct {
err error
pkg string
versions []string
}
items := strings.Split(pkg, "/")
dataMap := make([]*Result, len(items))
notify := make(chan struct{}, 1)
maxIndex := len(items) - 1
for i := maxIndex; i >= 1; i-- {
go func(i int) {
pkg := strings.Join(items[0:i+1], "/")
versions, err := ListGoPackageVersions(context.TODO(), log, pkg)
dataMap[maxIndex-i] = &Result{
pkg: pkg,
versions: versions,
err: err,
}
notify <- struct{}{}
}(i)
}
OutLoop:
for {
select {
case <-notify:
var errs error
for _, data := range dataMap {
if data == nil {
continue OutLoop
}
if data.err != nil {
errs = multierror.Append(errs, data.err)
}
if data.versions != nil {
return data.versions, nil
}
}
return nil, errs
case <-ctx.Done():
return nil, ctx.Err()
}
}
}

View file

@ -0,0 +1,27 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPluginmanager(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pluginmanager Suite")
}

View file

@ -0,0 +1,73 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager_test
import (
"context"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/storyicon/powerproto/pkg/component/pluginmanager"
"github.com/storyicon/powerproto/pkg/util/logger"
)
var _ = Describe("Pluginmanager", func() {
cfg := pluginmanager.NewConfig()
cfg.StorageDir, _ = filepath.Abs("./tests")
const pluginPkg = "google.golang.org/protobuf/cmd/protoc-gen-go"
var manager pluginmanager.PluginManager
It("should able to init", func() {
pluginManager, err := pluginmanager.NewPluginManager(cfg, logger.NewDefault("pluginmanager"))
Expect(err).To(BeNil())
Expect(pluginManager).To(Not(BeNil()))
manager = pluginManager
})
It("should able to install protoc", func() {
versions, err := manager.ListProtocVersions(context.TODO())
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetProtocLatestVersion(context.TODO())
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallProtoc(context.TODO(), latestVersion)
Expect(err).To(BeNil())
exists, local, err := manager.IsProtocInstalled(context.TODO(), latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
It("should able to install plugin", func() {
versions, err := manager.ListPluginVersions(context.TODO(), pluginPkg)
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetPluginLatestVersion(context.TODO(), pluginPkg)
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallPlugin(context.TODO(), pluginPkg, latestVersion)
Expect(err).To(BeNil())
Expect(len(local) > 0).To(BeTrue())
exists, local, err := manager.IsPluginInstalled(context.TODO(), pluginPkg, latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
})

View file

@ -0,0 +1,186 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/mholt/archiver"
"github.com/pkg/errors"
"github.com/storyicon/powerproto/pkg/util"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
// ProtocRelease defines the release of protoc
type ProtocRelease struct {
workspace string
}
// GetIncludePath is used to get the include path
func (p *ProtocRelease) GetIncludePath() string {
return filepath.Join(p.workspace, "include")
}
// GetProtocPath is used to get the protoc path
func (p *ProtocRelease) GetProtocPath() string {
return filepath.Join(p.workspace, "bin", util.GetBinaryFileName("protoc"))
}
// Clear is used to clear the workspace
func (p *ProtocRelease) Clear() error {
return os.RemoveAll(p.workspace)
}
// GetProtocRelease is used to download protoc release
func GetProtocRelease(ctx context.Context, version string) (*ProtocRelease, error) {
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
suffix, err := inferProtocReleaseSuffix()
if err != nil {
return nil, err
}
filename := fmt.Sprintf("protoc-%s-%s.zip", version, suffix)
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/"+
"releases/download/v%s/%s", version, filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: url,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &ProtocRelease{
workspace: workspace,
}, nil
}
// IsProtocInstalled is used to check whether the protoc version is installed
func IsProtocInstalled(ctx context.Context, storageDir string, version string) (bool, string, error) {
local := PathForProtoc(storageDir, version)
exists, err := util.IsFileExists(local)
if err != nil {
return false, "", err
}
return exists, local, nil
}
// ErrGitList defines the git list error
type ErrGitList struct {
*command.ErrCommandExec
}
// ListGitTags is used to list the git tags of specified repository
func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string, error) {
data, err := command.Execute(ctx, log, "", "git", []string{
"ls-remote", "--tags", "--refs", "--sort", "version:refname", repo,
}, nil)
if err != nil {
return nil, &ErrGitList{
ErrCommandExec: err,
}
}
var tags []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
if strings.HasPrefix(f[1], "refs/tags/") {
tags = append(tags, strings.TrimPrefix(f[1], "refs/tags/"))
}
}
return tags, nil
}
func inferProtocReleaseSuffix() (string, error) {
goos := strings.ToLower(runtime.GOOS)
arch := strings.ToLower(runtime.GOARCH)
switch goos {
case "linux":
switch arch {
case "arm64":
return "linux-aarch_64", nil
case "ppc64le":
return "linux-ppcle_64", nil
case "s390x":
return "linux-s390_64", nil
case "386":
return "linux-x86_32", nil
case "amd64":
return "linux-x86_64", nil
}
case "darwin":
return "osx-x86_64", nil
case "windows":
switch arch {
case "386":
return "win32", nil
case "amd64":
return "win64", nil
}
}
return "", errors.New("protoc did not release on this platform")
}
func downloadFile(resp *http.Response, destination string) error {
if err := os.MkdirAll(filepath.Dir(destination), fs.ModePerm); err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.Errorf("unexpected code %d for url: %s", resp.StatusCode, resp.Request.URL.String())
}
file, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fs.ModePerm)
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
return nil
}