feat(*): support repositories

Signed-off-by: storyicon <yuanchao@bilibili.com>
This commit is contained in:
storyicon 2021-07-23 14:56:00 +08:00
parent 1c73b92f0f
commit 9a99c53c5b
No known key found for this signature in database
GPG key ID: 245915D985F966CF
18 changed files with 542 additions and 255 deletions

View file

@ -44,7 +44,7 @@ PowerProto is used to solve the following three main problems:
5. support batch and recursive compilation of proto files to improve efficiency.
6. cross-platform support PostAction, you can perform some routine operations (such as replacing "omitempty" in all generated files) after the compilation.
7. support PostShell, execute specific shell scripts after the compilation.
8. one-click installation and version control of google apis。
8. one-click installation and version control of google apis and gogo protobuf etc
## Installation and Dependencies
@ -179,6 +179,8 @@ protocWorkDir: ""
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@latest
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -188,6 +190,7 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
```
@ -222,6 +225,7 @@ $POWERPROTO_HOME/protoc/3.17.3/protoc --go_out=. \
--proto_path=/mnt/data/hello \
--proto_path=$GOPATH \
--proto_path=$POWERPROTO_HOME/include \
--proto_path=$POWERPROTO_HOME/gits/75e9812478607db997376ccea247dd6928f70f45/github.com/googleapis/googleapis \
--plugin=protoc-gen-go=$POWERPROTO_HOME/plugins/google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1/protoc-gen-go \
--plugin=protoc-gen-go-grpc=$POWERPROTO_HOME/plugins/google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0/protoc-gen-go-grpc \
/mnt/data/hello/apis/hello.proto
@ -249,9 +253,17 @@ protoc: 3.17.3
# the default is the directory where the config file is located.
# support mixed environment variables in path, such as $GOPATH
protocWorkDir: ""
# optional. If you need to use googleapis, you should fill in the commit id of googleapis here.
# You can fill in the latest, it will be automatically converted to the latest version.
googleapis: 75e9812478607db997376ccea247dd6928f70f45
# optional. define dependent Git repositories
# Generally used for dependency control of public protobuf libraries
repositories:
# Definition depends on the 27156597fdf4fb77004434d4409154a230dc9a32 version of https://github.com/googleapis/googleapis
# and defines its name as GOOGLE_APIS
# It can be referenced in importPaths by $GOOGLE_APIS
GOOGLE_APIS: https://github.com/googleapis/googleapis@27156597fdf4fb77004434d4409154a230dc9a32
# Definition depends on the 226206f39bd7276e88ec684ea0028c18ec2c91ae version of https://github.com/gogo/protobuf
# and defines its name as GOGO_PROTOBUF
# It can be referenced in the importPaths by $GOGO_PROTOBUF
GOGO_PROTOBUF: https://github.com/gogo/protobuf@226206f39bd7276e88ec684ea0028c18ec2c91ae
# required. it is used to describe which plug-ins are required for compilation
plugins:
# the name, path, and version number of the plugin.
@ -280,8 +292,10 @@ importPaths:
# Special variables. Reference to the directory where the proto file to be compiled is located
# For example, if /a/b/data.proto is to be compiled, then the /a/b directory will be automatically referenced
- $SOURCE_RELATIVE
# Special variables. Will be replaced with the local path to the version of google apis specified by the googleapis field
- $POWERPROTO_GOOGLEAPIS
# References GOOGLE_APIS as defined in repositories
- $GOOGLE_APIS/github.com/googleapis/googleapis
# References GOGO_PROTOBUF as defined in repositories
- $GOGO_PROTOBUF
# optional. The operation is executed after compilation.
# its working directory is the directory where the config file is located.
# postActions is cross-platform compatible.
@ -312,10 +326,11 @@ scopes:
- ./apis1
protoc: v3.17.3
protocWorkDir: ""
googleapis: 75e9812478607db997376ccea247dd6928f70f45
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@v1.25.0
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -325,6 +340,7 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
@ -334,10 +350,11 @@ scopes:
- ./apis2
protoc: v3.17.3
protocWorkDir: ""
googleapis: 75e9812478607db997376ccea247dd6928f70f45
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.0
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -347,12 +364,11 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
```
### PostAction
PostAction allows to perform specific actions after all proto files have been compiled. In contrast to `PostShell`, it is cross-platform supported.

View file

@ -44,7 +44,7 @@ PowerProto主要用于解决下面三个问题
5. 支持批量、递归编译proto文件提高效率。
6. 跨平台支持PostAction可以在编译完成之后执行一些常规操作比如替换掉所有生成文件中的"omitempty")。
7. 支持PostShell在编译完成之后执行特定的shell脚本。
8. 支持 google api 的一键安装与版本控制。
8. 支持 google api, gogo protobuf 等的一键安装与版本控制。
## 安装与依赖
@ -171,6 +171,8 @@ protocWorkDir: ""
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@latest
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -180,6 +182,7 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
```
@ -214,6 +217,7 @@ $POWERPROTO_HOME/protoc/3.17.3/protoc --go_out=. \
--proto_path=/mnt/data/hello \
--proto_path=$GOPATH \
--proto_path=$POWERPROTO_HOME/include \
--proto_path=$POWERPROTO_HOME/gits/75e9812478607db997376ccea247dd6928f70f45/github.com/googleapis/googleapis \
--plugin=protoc-gen-go=$POWERPROTO_HOME/plugins/google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1/protoc-gen-go \
--plugin=protoc-gen-go-grpc=$POWERPROTO_HOME/plugins/google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0/protoc-gen-go-grpc
/mnt/data/hello/apis/hello.proto
@ -239,9 +243,17 @@ protoc: 3.17.3
# 选填执行protoc命令的工作目录默认是配置文件所在目录
# 支持路径中混用环境变量,比如$GOPATH
protocWorkDir: ""
# 选填,如果需要使用 googleapis你应该在这里填写googleapis的commit id
# 可以填 latest会自动转换成最新的版本
googleapis: 75e9812478607db997376ccea247dd6928f70f45
# 选填定义依赖的Git存储库
# 一般用于公共的protobuf库的依赖控制
repositories:
# 定义依赖 27156597fdf4fb77004434d4409154a230dc9a32 版本的 https://github.com/googleapis/googleapis
# 并且定义其名字为 GOOGLE_APIS
# 在 importPaths 中可以通过 $GOOGLE_APIS 来引用它
GOOGLE_APIS: https://github.com/googleapis/googleapis@27156597fdf4fb77004434d4409154a230dc9a32
# 定义依赖 226206f39bd7276e88ec684ea0028c18ec2c91ae 版本的 https://github.com/gogo/protobuf
# 并且定义其名字为 GOGO_PROTOBUF
# 在 importPaths 中可以通过 $GOGO_PROTOBUF 来引用它
GOGO_PROTOBUF: https://github.com/gogo/protobuf@226206f39bd7276e88ec684ea0028c18ec2c91ae
# 必填代表scope匹配的目录中的proto文件在编译时需要用到哪些插件
plugins:
# 插件的名字、路径以及版本号。
@ -268,8 +280,10 @@ importPaths:
# 特殊变量。引用待编译的proto文件所在的目录
# 比如将要编译 /a/b/data.proto那么 /a/b 目录将会被自动引用
- $SOURCE_RELATIVE
# 特殊变量。引用googleapis字段所指定的版本的google apis
- $POWERPROTO_GOOGLEAPIS
# 引用 repositories 中的 GOOGLE_APIS
- $GOOGLE_APIS/github.com/googleapis/googleapis
# 引用 repositories 中的 GOGO_PROTOBUF
- $GOGO_PROTOBUF
# 选填,构建完成之后执行的操作,工作目录是配置文件所在目录
# postActions是跨平台兼容的
# 注意,必须在 powerproto build 时附加 -p 参数才会执行配置文件中的postActions
@ -299,10 +313,11 @@ scopes:
- ./apis1
protoc: v3.17.3
protocWorkDir: ""
googleapis: 75e9812478607db997376ccea247dd6928f70f45
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@v1.25.0
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -312,6 +327,7 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
@ -321,10 +337,11 @@ scopes:
- ./apis2
protoc: v3.17.3
protocWorkDir: ""
googleapis: 75e9812478607db997376ccea247dd6928f70f45
plugins:
protoc-gen-go: google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.0
protoc-gen-go-grpc: google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
repositories:
GOOGLE_APIS: https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45
options:
- --go_out=.
- --go_opt=paths=source_relative
@ -334,6 +351,7 @@ importPaths:
- .
- $GOPATH
- $POWERPROTO_INCLUDE
- $GOOGLE_APIS/github.com/googleapis/googleapis
postActions: []
postShell: ""
```

View file

@ -15,6 +15,8 @@
package build
import (
"fmt"
"github.com/AlecAivazis/survey/v2"
"github.com/spf13/cobra"
@ -27,6 +29,7 @@ import (
// UserPreference defines the model of user preference
type UserPreference struct {
Plugins []string `survey:"plugins"`
Repositories []string `survey:"repositories"`
}
// GetUserPreference is used to get user preference
@ -40,7 +43,25 @@ func GetUserPreference() (*UserPreference, error) {
Options: GetWellKnownPluginsOptionValues(),
},
},
{
Name: "repositories",
Prompt: &survey.MultiSelect{
Message: "select repositories to use. Later, you can also manually add in the configuration file",
Options: GetWellKnownRepositoriesOptionValues(),
},
},
}, &preference)
if len(preference.Plugins) == 0 {
preference.Plugins = []string{
GetPluginProtocGenGo().GetOptionsValue(),
GetPluginProtocGenGoGRPC().GetOptionsValue(),
}
}
if len(preference.Repositories) == 0 {
preference.Repositories = []string{
GetRepositoryGoogleAPIs().GetOptionsValue(),
}
}
return &preference, err
}
@ -51,22 +72,13 @@ func GetDefaultConfig() *configs.Config {
"./",
},
Protoc: "latest",
GoogleAPIs: "75e9812478607db997376ccea247dd6928f70f45",
Plugins: map[string]string{
"protoc-gen-go": "google.golang.org/protobuf/cmd/protoc-gen-go@latest",
"protoc-gen-go-grpc": "google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest",
},
Options: []string{
"--go_out=.",
"--go_opt=paths=source_relative",
"--go-grpc_out=.",
"--go-grpc_opt=paths=source_relative",
},
Plugins: map[string]string{},
Repositories: map[string]string{},
Options: []string{},
ImportPaths: []string{
".",
"$GOPATH",
consts.KeyPowerProtoInclude,
consts.KeyPowerProtoGoogleAPIs,
consts.KeySourceRelative,
},
}
@ -95,18 +107,18 @@ func CommandInit(log logger.Logger) *cobra.Command {
return
}
config := GetDefaultConfig()
if len(preference.Plugins) != 0 {
var compileOptions []string
plugins := map[string]string{}
for _, val := range preference.Plugins {
plugin, ok := GetPluginFromOptionsValue(val)
if ok {
plugins[plugin.Name] = plugin.Pkg
compileOptions = append(compileOptions, plugin.Options...)
if plugin, ok := GetPluginFromOptionsValue(val); ok {
config.Plugins[plugin.Name] = plugin.Pkg
config.Options = append(config.Options, plugin.Options...)
}
}
config.Plugins = plugins
config.Options = compileOptions
fmt.Println(">>>>>>>>>>>>>>", preference.Repositories)
for _, val := range preference.Repositories {
if repo, ok := GetRepositoryFromOptionsValue(val); ok {
config.Repositories[repo.Name] = repo.Pkg
config.ImportPaths = append(config.ImportPaths, repo.ImportPaths...)
}
}
if err := configs.SaveConfigs(consts.ConfigFileName, config); err != nil {
log.LogFatal(nil, "failed to save config: %s", err)

View file

@ -20,35 +20,50 @@ import (
// Plugin defines the plugin options
type Plugin struct {
OptionsValue string
Name string
Pkg string
Options []string
}
// String implements the standard string interface
func (options *Plugin) String() string {
return fmt.Sprintf("%s: %s", options.Name, options.Pkg)
// GetOptionsValue is used to get options value of plugin
func (plugin *Plugin) GetOptionsValue() string {
if plugin.OptionsValue != "" {
return plugin.OptionsValue
}
return fmt.Sprintf("%s: %s", plugin.Name, plugin.Pkg)
}
// GetWellKnownPlugins is used to get well known plugins
func GetWellKnownPlugins() []*Plugin {
return []*Plugin{
{
// GetPluginProtocGenGo is used to get protoc-gen-go plugin
func GetPluginProtocGenGo() *Plugin {
return &Plugin{
Name: "protoc-gen-go",
Pkg: "google.golang.org/protobuf/cmd/protoc-gen-go@latest",
Options: []string{
"--go_out=.",
"--go_opt=paths=source_relative",
},
},
{
}
}
// GetPluginProtocGenGoGRPC is used to get protoc-gen-go-grpc plugin
func GetPluginProtocGenGoGRPC() *Plugin {
return &Plugin{
Name: "protoc-gen-go-grpc",
Pkg: "google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest",
Options: []string{
"--go-grpc_out=.",
"--go-grpc_opt=paths=source_relative",
},
},
}
}
// GetWellKnownPlugins is used to get well known plugins
func GetWellKnownPlugins() []*Plugin {
return []*Plugin{
GetPluginProtocGenGo(),
GetPluginProtocGenGoGRPC(),
{
Name: "protoc-gen-grpc-gateway",
Pkg: "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@latest",
@ -56,6 +71,15 @@ func GetWellKnownPlugins() []*Plugin {
"--grpc-gateway_out=.",
},
},
{
OptionsValue: "protoc-gen-go-grpc (SupportPackageIsVersion6)",
Name: "protoc-gen-go-grpc",
Pkg: "google.golang.org/grpc/cmd/protoc-gen-go-grpc@ad51f572fd270f2323e3aa2c1d2775cab9087af2",
Options: []string{
"--go-grpc_out=.",
"--go-grpc_opt=paths=source_relative",
},
},
{
Name: "protoc-gen-openapiv2",
Pkg: "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@latest",
@ -98,7 +122,7 @@ func GetWellKnownPlugins() []*Plugin {
func GetPluginFromOptionsValue(val string) (*Plugin, bool) {
plugins := GetWellKnownPlugins()
for _, plugin := range plugins {
if plugin.String() == val {
if plugin.GetOptionsValue() == val {
return plugin, true
}
}
@ -110,7 +134,7 @@ func GetWellKnownPluginsOptionValues() []string {
plugins := GetWellKnownPlugins()
packages := make([]string, 0, len(plugins))
for _, plugin := range plugins {
packages = append(packages, plugin.String())
packages = append(packages, plugin.GetOptionsValue())
}
return packages
}

View file

@ -0,0 +1,88 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package build
import (
"fmt"
"strings"
)
// Repository defines the plugin options
type Repository struct {
OptionsValue string
Name string
Pkg string
ImportPaths []string
}
// OptionsValue is used to return the options value
func (repo *Repository) GetOptionsValue() string {
if repo.OptionsValue != "" {
return repo.OptionsValue
}
return fmt.Sprintf("%s: %s", strings.ToLower(repo.Name), repo.Pkg)
}
// GetWellKnownRepositories is used to get well known plugins
func GetWellKnownRepositories() []*Repository {
return []*Repository{
GetRepositoryGoogleAPIs(),
GetRepositoryGoGoProtobuf(),
}
}
// GetRepositoryGoGoProtobuf is used to get gogo protobuf repository
func GetRepositoryGoGoProtobuf() *Repository {
return &Repository{
Name: "GOGO_PROTOBUF",
Pkg: "https://github.com/gogo/protobuf@226206f39bd7276e88ec684ea0028c18ec2c91ae",
ImportPaths: []string{
"$GOGO_PROTOBUF",
},
}
}
// GetRepositoryGoogleAPIs is used to get google apis repository
func GetRepositoryGoogleAPIs() *Repository {
return &Repository{
Name: "GOOGLE_APIS",
Pkg: "https://github.com/googleapis/googleapis@75e9812478607db997376ccea247dd6928f70f45",
ImportPaths: []string{
"$GOOGLE_APIS/github.com/googleapis/googleapis",
},
}
}
// GetRepositoryFromOptionsValue is used to get plugin by option value
func GetRepositoryFromOptionsValue(val string) (*Repository, bool) {
repositories := GetWellKnownRepositories()
for _, repo := range repositories {
if repo.GetOptionsValue() == val {
return repo, true
}
}
return nil, false
}
// GetWellKnownRepositoriesOptionValues is used to get option values of well known plugins
func GetWellKnownRepositoriesOptionValues() []string {
repos := GetWellKnownRepositories()
packages := make([]string, 0, len(repos))
for _, repo := range repos {
packages = append(packages, repo.GetOptionsValue())
}
return packages
}

View file

@ -48,7 +48,7 @@ func tidy(ctx context.Context,
if err := bootstraps.StepInstallProtoc(ctx, pluginManager, configItems); err != nil {
return err
}
if err := bootstraps.StepInstallGoogleAPIs(ctx, pluginManager, configItems); err != nil {
if err := bootstraps.StepInstallRepositories(ctx, pluginManager, configItems); err != nil {
return err
}
if err := bootstraps.StepInstallPlugins(ctx, pluginManager, configItems); err != nil {

View file

@ -67,54 +67,61 @@ func StepLookUpConfigs(
return configItems, nil
}
// StepInstallGoogleAPIs is used to install google apis
func StepInstallGoogleAPIs(ctx context.Context,
// StepInstallRepositories is used to install repositories
func StepInstallRepositories(ctx context.Context,
pluginManager pluginmanager.PluginManager,
configItems []configs.ConfigItem) error {
deduplicate := map[string]struct{}{}
for _, config := range configItems {
version := config.Config().GoogleAPIs
if version != "" {
deduplicate[version] = struct{}{}
for _, pkg := range config.Config().Repositories {
deduplicate[pkg] = struct{}{}
}
}
if len(deduplicate) == 0 {
return nil
}
progress := progressbar.GetProgressBar(ctx, len(deduplicate))
progress.SetPrefix("Install googleapis")
progress.SetPrefix("Install repositories")
versionsMap := map[string]struct{}{}
for version := range deduplicate {
repoMap := map[string]struct{}{}
for pkg := range deduplicate {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("invalid format: %s, should in path@version format", pkg)
}
if version == "latest" {
progress.SetSuffix("query latest version of googleapis")
latestVersion, err := pluginManager.GetGoogleAPIsLatestVersion(ctx)
progress.SetSuffix("query latest version of %s", path)
latestVersion, err := pluginManager.GetGitRepoLatestVersion(ctx, path)
if err != nil {
return errors.Wrap(err, "failed to list googleapis versions")
return errors.Wrapf(err, "failed to query latest version of %s", path)
}
version = latestVersion
pkg = util.JoinGoPackageVersion(path, version)
}
progress.SetSuffix("check cache of googleapis %s", version)
exists, _, err := pluginManager.IsGoogleAPIsInstalled(ctx, version)
progress.SetSuffix("check cache of %s", pkg)
exists, _, err := pluginManager.IsGitRepoInstalled(ctx, path, version)
if err != nil {
return err
}
if exists {
progress.SetSuffix("the %s version of googleapis is already cached", version)
progress.SetSuffix("the %s version of %s is already cached", version, path)
} else {
progress.SetSuffix("install %s version of googleapis", version)
_, err = pluginManager.InstallGoogleAPIs(ctx, version)
progress.SetSuffix("install %s version of %s", version, path)
_, err = pluginManager.InstallGitRepo(ctx, path, version)
if err != nil {
return err
}
progress.SetSuffix("the %s version of googleapis is installed", version)
progress.SetSuffix("the %s version of %s is installed", version, path)
}
versionsMap[version] = struct{}{}
repoMap[pkg] = struct{}{}
progress.Incr()
}
progress.SetSuffix("all repositories have been installed")
progress.Wait()
fmt.Println("the following versions of googleapis will be used:", util.SetToSlice(versionsMap))
fmt.Println("the following versions of googleapis will be used:")
for pkg := range repoMap {
fmt.Printf(" %s\r\n", pkg)
}
return nil
}
@ -320,7 +327,7 @@ func Compile(ctx context.Context, targets []string) error {
if err := StepInstallProtoc(ctx, pluginManager, configItems); err != nil {
return err
}
if err := StepInstallGoogleAPIs(ctx, pluginManager, configItems); err != nil {
if err := StepInstallRepositories(ctx, pluginManager, configItems); err != nil {
return err
}
if err := StepInstallPlugins(ctx, pluginManager, configItems); err != nil {

View file

@ -95,6 +95,21 @@ func StepTidyConfigFile(ctx context.Context,
item.Protoc = version
cleanable = true
}
for name, pkg := range item.Repositories {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("invalid package format: %s, should be in path@version format", pkg)
}
if version == "latest" {
progress.SetSuffix("query latest version of %s", path)
version, err := pluginManager.GetGitRepoLatestVersion(ctx, path)
if err != nil {
return err
}
item.Repositories[name] = util.JoinGoPackageVersion(path, version)
cleanable = true
}
}
for name, pkg := range item.Plugins {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {

View file

@ -47,7 +47,8 @@ type BasicCompiler struct {
protocPath string
arguments []string
googleapisPath string
// map[name]local
repositories map[string]string
dir string
}
@ -73,7 +74,10 @@ func NewBasicCompiler(
config: config,
pluginManager: pluginManager,
}
if err := basic.calcGoogleAPIs(ctx); err != nil {
basic.repositories = map[string]string{
consts.KeyNamePowerProtocInclude: basic.pluginManager.IncludePath(ctx),
}
if err := basic.calcRepositories(ctx); err != nil {
return nil, err
}
if err := basic.calcProto(ctx); err != nil {
@ -98,7 +102,7 @@ func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error
consts.KeySourceRelative) {
arguments = append(arguments, "--proto_path="+filepath.Dir(protoFilePath))
}
arguments = util.DeduplicateSliceStably(arguments)
arguments = append(arguments, protoFilePath)
_, err := command.Execute(ctx,
b.Logger, b.dir, b.protocPath, arguments, nil)
@ -117,7 +121,7 @@ func (b *BasicCompiler) GetConfig(ctx context.Context) configs.ConfigItem {
func (b *BasicCompiler) calcDir(ctx context.Context) error {
if dir := b.config.Config().ProtocWorkDir; dir != "" {
dir = util.RenderPathWithEnv(dir)
dir = util.RenderPathWithEnv(dir, nil)
if !filepath.IsAbs(dir) {
dir = filepath.Join(b.config.Path(), dir)
}
@ -128,30 +132,6 @@ func (b *BasicCompiler) calcDir(ctx context.Context) error {
return nil
}
func (b *BasicCompiler) calcGoogleAPIs(ctx context.Context) error {
cfg := b.config.Config()
commitId := cfg.GoogleAPIs
if commitId == "" {
return nil
}
if !util.Contains(cfg.ImportPaths, consts.KeyPowerProtoGoogleAPIs) {
return nil
}
if commitId == "latest" {
latestVersion, err := b.pluginManager.GetGoogleAPIsLatestVersion(ctx)
if err != nil {
return err
}
commitId = latestVersion
}
local, err := b.pluginManager.InstallGoogleAPIs(ctx, commitId)
if err != nil {
return err
}
b.googleapisPath = local
return nil
}
func (b *BasicCompiler) calcProto(ctx context.Context) error {
cfg := b.config
protocVersion := cfg.Config().Protoc
@ -179,17 +159,10 @@ func (b *BasicCompiler) calcArguments(ctx context.Context) error {
// build import paths
Loop:
for _, path := range cfg.Config().ImportPaths {
switch path {
case consts.KeyPowerProtoInclude:
path = b.pluginManager.IncludePath(ctx)
case consts.KeyPowerProtoGoogleAPIs:
if b.googleapisPath != "" {
path = b.googleapisPath
}
case consts.KeySourceRelative:
if path == consts.KeySourceRelative {
continue Loop
}
path = util.RenderPathWithEnv(path)
path = util.RenderPathWithEnv(path, b.repositories)
if !filepath.IsAbs(path) {
path = filepath.Join(dir, path)
}
@ -219,3 +192,26 @@ Loop:
b.arguments = arguments
return nil
}
func (b *BasicCompiler) calcRepositories(ctx context.Context) error {
cfg := b.config
for name, pkg := range cfg.Config().Repositories {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("failed to parse: %s", pkg)
}
if version == "latest" {
latestVersion, err := b.pluginManager.GetGitRepoLatestVersion(ctx, path)
if err != nil {
return err
}
version = latestVersion
}
local, err := b.pluginManager.InstallGitRepo(ctx, path, version)
if err != nil {
return errors.Wrap(err, "failed to get plugin path")
}
b.repositories[name] = local
}
return nil
}

View file

@ -16,8 +16,15 @@ package pluginmanager
import (
"context"
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/mholt/archiver"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -89,3 +96,62 @@ func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string,
}
return tags, nil
}
// GithubArchive is github archive
type GithubArchive struct {
uri string
commit string
workspace string
}
// GetGithubArchive is used to download github archive
func GetGithubArchive(ctx context.Context, uri string, commitId string) (*GithubArchive, error) {
filename := fmt.Sprintf("%s.zip", commitId)
addr := fmt.Sprintf("%s/archive/%s", uri, filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, addr, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
}
}
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &GithubArchive{
uri: uri,
commit: commitId,
workspace: workspace,
}, nil
}
// GetLocalDir is used to get local dir of archive
func (c *GithubArchive) GetLocalDir() string {
dir := path.Base(c.uri) + "-" + c.commit
return filepath.Join(c.workspace, dir)
}
// Clear is used to clear the workspace
func (c *GithubArchive) Clear() error {
return os.RemoveAll(c.workspace)
}

View file

@ -1,81 +0,0 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"github.com/mholt/archiver"
)
// GoogleAPIRelease defines the release of google api
type GoogleAPIRelease struct {
workspace string
commit string
}
// GetDir is used to get the dir of google api
func (p *GoogleAPIRelease) GetDir() string {
return filepath.Join(p.workspace, "googleapis-"+p.commit)
}
// Clear is used to clear the workspace
func (p *GoogleAPIRelease) Clear() error {
return os.RemoveAll(p.workspace)
}
// GetGoogleAPIRelease is used to get the release of google api
func GetGoogleAPIRelease(ctx context.Context, commitId string) (*GoogleAPIRelease, error) {
filename := fmt.Sprintf("%s.zip", commitId)
uri := fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s", filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &GoogleAPIRelease{
commit: commitId,
workspace: workspace,
}, nil
}

View file

@ -41,16 +41,17 @@ type PluginManager interface {
// InstallPlugin is used to install plugin
InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
GetGoogleAPIsLatestVersion(ctx context.Context) (string, error)
// InstallGoogleAPIs is used to install google apis
InstallGoogleAPIs(ctx context.Context, commitId string) (local string, err error)
// ListGoogleAPIsVersions is used to list protoc version
ListGoogleAPIsVersions(ctx context.Context) ([]string, error)
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error)
// GoogleAPIsPath returns the googleapis path
GoogleAPIsPath(ctx context.Context, commitId string) string
// GetGitRepoLatestVersion is used to get the latest version of google apis
GetGitRepoLatestVersion(ctx context.Context, uri string) (string, error)
// InstallGitRepo is used to install google apis
InstallGitRepo(ctx context.Context, uri string, commitId string) (local string, err error)
// ListGitRepoVersions is used to list protoc version
ListGitRepoVersions(ctx context.Context, uri string) ([]string, error)
// IsGitRepoInstalled is used to check whether the protoc is installed
IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error)
// GitRepoPath returns the googleapis path
GitRepoPath(ctx context.Context, commitId string) string
// GetProtocLatestVersion is used to get the latest version of protoc
GetProtocLatestVersion(ctx context.Context) (string, error)
// ListProtocVersions is used to list protoc version
@ -140,9 +141,9 @@ func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, ver
return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
}
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListGoogleAPIsVersions(ctx)
// GetGitRepoLatestVersion is used to get the latest version of google apis
func (b *BasicPluginManager) GetGitRepoLatestVersion(ctx context.Context, url string) (string, error) {
versions, err := b.ListGitRepoVersions(ctx, url)
if err != nil {
return "", err
}
@ -152,60 +153,67 @@ func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (st
return versions[len(versions)-1], nil
}
// InstallGoogleAPIs is used to install google apis
func (b *BasicPluginManager) InstallGoogleAPIs(ctx context.Context, commitId string) (string, error) {
// InstallGitRepo is used to install google apis
func (b *BasicPluginManager) InstallGitRepo(ctx context.Context, uri string, commitId string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
exists, local, err := b.IsGitRepoInstalled(ctx, uri, commitId)
if err != nil {
return "", err
}
if exists {
return local, nil
}
release, err := GetGoogleAPIRelease(ctx, commitId)
release, err := GetGithubArchive(ctx, uri, commitId)
if err != nil {
return "", err
}
defer release.Clear()
if err := util.CopyDirectory(release.GetDir(), local); err != nil {
codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
if err != nil {
return "", err
}
if err := util.CopyDirectory(release.GetLocalDir(), codePath); err != nil {
return "", err
}
return local, nil
}
// ListGoogleAPIsVersions is used to list protoc version
func (b *BasicPluginManager) ListGoogleAPIsVersions(ctx context.Context) ([]string, error) {
// ListGitRepoVersions is used to list protoc version
func (b *BasicPluginManager) ListGitRepoVersions(ctx context.Context, uri string) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions["googleapis"]
versions, ok := b.versions[uri]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListGitCommitIds(ctx, b.Logger, consts.GoogleAPIsRepository)
versions, err := ListGitCommitIds(ctx, b.Logger, uri)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions["googleapis"] = versions
b.versions[uri] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error) {
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
return exists, local, err
// IsGitRepoInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error) {
codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
if err != nil {
return false, "", err
}
exists, err := util.IsDirExists(codePath)
return exists, PathForGitRepos(b.storageDir, commitId), err
}
// GoogleAPIsPath returns the googleapis path
func (b *BasicPluginManager) GoogleAPIsPath(ctx context.Context, commitId string) string {
return PathForGoogleAPIs(b.storageDir, commitId)
// GitRepoPath returns the googleapis path
func (b *BasicPluginManager) GitRepoPath(ctx context.Context, commitId string) string {
return PathForGitRepos(b.storageDir, commitId)
}
// IsProtocInstalled is used to check whether the protoc is installed

View file

@ -15,6 +15,7 @@
package pluginmanager
import (
"net/url"
"path"
"path/filepath"
@ -46,9 +47,19 @@ func GetPluginPath(path string, version string) (string, error) {
return filepath.Join(enc + "@" + encVer), nil
}
// PathForGoogleAPIs is used to get the google apis path
func PathForGoogleAPIs(storageDir string, commitId string) string {
return filepath.Join(storageDir, "googleapis", commitId)
// PathForGitReposCode returns the code path for git repos
func PathForGitReposCode(storageDir string, uri string, commitId string) (string, error) {
parsed, err := url.Parse(uri)
if err != nil {
return "", err
}
dir := parsed.Host + parsed.Path
return filepath.Join(PathForGitRepos(storageDir, commitId), dir), nil
}
// PathForGitRepos is used to get the git repo local path
func PathForGitRepos(storageDir string, commitId string) string {
return filepath.Join(storageDir, "gits", commitId)
}
// PathForPluginDir is used to get the local directory where the specified version plug-in should be stored

View file

@ -70,20 +70,20 @@ var _ = Describe("Pluginmanager", func() {
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
It("should able to install googleapis", func() {
versions, err := manager.ListGoogleAPIsVersions(context.TODO())
It("should able to install git repos", func() {
const uri = "https://github.com/gogo/protobuf"
versions, err := manager.ListGitRepoVersions(context.TODO(), uri)
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetGoogleAPIsLatestVersion(context.TODO())
latestVersion, err := manager.GetGitRepoLatestVersion(context.TODO(), uri)
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallGoogleAPIs(context.TODO(), latestVersion)
local, err := manager.InstallGitRepo(context.TODO(), uri, latestVersion)
Expect(err).To(BeNil())
Expect(len(local) > 0).To(BeTrue())
exists, local, err := manager.IsGoogleAPIsInstalled(context.TODO(), latestVersion)
exists, local, err := manager.IsGitRepoInstalled(context.TODO(), uri, latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())

View file

@ -29,10 +29,10 @@ import (
// Config defines the config model
type Config struct {
Scopes []string `json:"scopes" yaml:"scopes"`
GoogleAPIs string `json:"googleapis" yaml:"googleapis"`
Protoc string `json:"protoc" yaml:"protoc"`
ProtocWorkDir string `json:"protocWorkDir" yaml:"protocWorkDir"`
Plugins map[string]string `json:"plugins" yaml:"plugins"`
Repositories map[string]string `json:"repositories" yaml:"repositories"`
Options []string `json:"options" yaml:"options"`
ImportPaths []string `json:"importPaths" yaml:"importPaths"`
PostActions []*PostAction `json:"postActions" yaml:"postActions"`

View file

@ -27,10 +27,10 @@ import (
const (
// ConfigFileName defines the config file name
ConfigFileName = "powerproto.yaml"
// KeyNamePowerProtocInclude is the key name of powerproto default include
KeyNamePowerProtocInclude = "POWERPROTO_INCLUDE"
// The default include can be referenced by this key in import paths
KeyPowerProtoInclude = "$POWERPROTO_INCLUDE"
// The googleapis can be referenced by this key in import paths
KeyPowerProtoGoogleAPIs = "$POWERPROTO_GOOGLEAPIS"
KeyPowerProtoInclude = "$" + KeyNamePowerProtocInclude
// KeySourceRelative can be specified in import paths to refer to
// the folder where the current proto file is located
KeySourceRelative = "$SOURCE_RELATIVE"

View file

@ -24,6 +24,19 @@ import (
"syscall"
)
// DeduplicateSlice is used to deduplicate slice items stably
func DeduplicateSliceStably(items []string) []string {
data := make([]string, 0, len(items))
deduplicate := map[string]struct{}{}
for _, val := range items {
if _, exists := deduplicate[val]; !exists {
deduplicate[val] = struct{}{}
data = append(data, val)
}
}
return data
}
// ContainsEmpty is used to check whether items contains empty string
func ContainsEmpty(items ...string) bool {
return Contains(items, "")
@ -75,10 +88,17 @@ func GetExitCode(err error) int {
var regexpEnvironmentVar = regexp.MustCompile(`\$[A-Za-z_]+`)
// RenderPathWithEnv is used to render path with environment
func RenderPathWithEnv(path string) string {
func RenderPathWithEnv(path string, ext map[string]string) string {
matches := regexpEnvironmentVar.FindAllString(path, -1)
for _, match := range matches {
path = strings.ReplaceAll(path, match, os.Getenv(match[1:]))
key := match[1:]
val := ext[key]
if val == "" {
val = os.Getenv(key)
}
if val != "" {
path = strings.ReplaceAll(path, match, val)
}
}
return filepath.Clean(path)
}

87
pkg/util/util_test.go Normal file
View file

@ -0,0 +1,87 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"path/filepath"
"reflect"
"testing"
)
func TestRenderPathWithEnv(t *testing.T) {
type args struct {
path string
ext map[string]string
}
tests := []struct {
name string
args args
want string
}{
{
args: args{
path: "$POWERPROTO_INCLUDE/protobuf",
ext: map[string]string{
"POWERPROTO_INCLUDE": "/mnt/powerproto/include",
},
},
want: filepath.Clean("/mnt/powerproto/include/protobuf"),
},
{
args: args{
path: "$POWERPROTO_INCLUDE/protobuf",
},
want: filepath.Clean("$POWERPROTO_INCLUDE/protobuf"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RenderPathWithEnv(tt.args.path, tt.args.ext); got != tt.want {
t.Errorf("RenderPathWithEnv() = %v, want %v", got, tt.want)
}
})
}
}
func TestDeduplicateSliceStably(t *testing.T) {
type args struct {
items []string
}
tests := []struct {
name string
args args
want []string
}{
{
args: args{
items: []string{"a", "B", "c", "B"},
},
want: []string{"a", "B", "c"},
},
{
args: args{
items: []string{"B", "c", "B", "a"},
},
want: []string{"B", "c", "a"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := DeduplicateSliceStably(tt.args.items); !reflect.DeepEqual(got, tt.want) {
t.Errorf("DeduplicateSliceStably() = %v, want %v", got, tt.want)
}
})
}
}