technofab/powerproto
1
0
Fork 0
mirror of https://github.com/TECHNOFAB11/powerproto.git synced 2025-12-11 23:50:04 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(*): support repositories

Browse source 
Signed-off-by: storyicon <yuanchao@bilibili.com>
This commit is contained in:
storyicon 2021-07-23 14:56:00 +08:00
parent 1c73b92f0f
commit 9a99c53c5b
No known key found for this signature in database
GPG key ID: 245915D985F966CF
18 changed files with 542 additions and 255 deletions
Download patch file Download diff file Expand all files Collapse all files

76
pkg/component/compilermanager/compiler.go
View file

@ -45,10 +45,11 @@ type BasicCompiler struct {
config configs.ConfigItem
pluginManager pluginmanager.PluginManager
protocPath string
arguments []string
googleapisPath string
dir string
protocPath string
arguments []string
// map[name]local
repositories map[string]string
dir string
}
// NewCompiler is used to create a compiler
@ -73,7 +74,10 @@ func NewBasicCompiler(
config: config,
pluginManager: pluginManager,
}
if err := basic.calcGoogleAPIs(ctx); err != nil {
basic.repositories = map[string]string{
consts.KeyNamePowerProtocInclude: basic.pluginManager.IncludePath(ctx),
}
if err := basic.calcRepositories(ctx); err != nil {
return nil, err
}
if err := basic.calcProto(ctx); err != nil {
@ -98,7 +102,7 @@ func (b *BasicCompiler) Compile(ctx context.Context, protoFilePath string) error
consts.KeySourceRelative) {
arguments = append(arguments, "--proto_path="+filepath.Dir(protoFilePath))
}
arguments = util.DeduplicateSliceStably(arguments)
arguments = append(arguments, protoFilePath)
_, err := command.Execute(ctx,
b.Logger, b.dir, b.protocPath, arguments, nil)
@ -117,7 +121,7 @@ func (b *BasicCompiler) GetConfig(ctx context.Context) configs.ConfigItem {
func (b *BasicCompiler) calcDir(ctx context.Context) error {
if dir := b.config.Config().ProtocWorkDir; dir != "" {
dir = util.RenderPathWithEnv(dir)
dir = util.RenderPathWithEnv(dir, nil)
if !filepath.IsAbs(dir) {
dir = filepath.Join(b.config.Path(), dir)
}
@ -128,30 +132,6 @@ func (b *BasicCompiler) calcDir(ctx context.Context) error {
return nil
}
func (b *BasicCompiler) calcGoogleAPIs(ctx context.Context) error {
cfg := b.config.Config()
commitId := cfg.GoogleAPIs
if commitId == "" {
return nil
}
if !util.Contains(cfg.ImportPaths, consts.KeyPowerProtoGoogleAPIs) {
return nil
}
if commitId == "latest" {
latestVersion, err := b.pluginManager.GetGoogleAPIsLatestVersion(ctx)
if err != nil {
return err
}
commitId = latestVersion
}
local, err := b.pluginManager.InstallGoogleAPIs(ctx, commitId)
if err != nil {
return err
}
b.googleapisPath = local
return nil
}
func (b *BasicCompiler) calcProto(ctx context.Context) error {
cfg := b.config
protocVersion := cfg.Config().Protoc
@ -179,17 +159,10 @@ func (b *BasicCompiler) calcArguments(ctx context.Context) error {
// build import paths
Loop:
for _, path := range cfg.Config().ImportPaths {
switch path {
case consts.KeyPowerProtoInclude:
path = b.pluginManager.IncludePath(ctx)
case consts.KeyPowerProtoGoogleAPIs:
if b.googleapisPath != "" {
path = b.googleapisPath
}
case consts.KeySourceRelative:
if path == consts.KeySourceRelative {
continue Loop
}
path = util.RenderPathWithEnv(path)
path = util.RenderPathWithEnv(path, b.repositories)
if !filepath.IsAbs(path) {
path = filepath.Join(dir, path)
}
@ -219,3 +192,26 @@ Loop:
b.arguments = arguments
return nil
}
func (b *BasicCompiler) calcRepositories(ctx context.Context) error {
cfg := b.config
for name, pkg := range cfg.Config().Repositories {
path, version, ok := util.SplitGoPackageVersion(pkg)
if !ok {
return errors.Errorf("failed to parse: %s", pkg)
}
if version == "latest" {
latestVersion, err := b.pluginManager.GetGitRepoLatestVersion(ctx, path)
if err != nil {
return err
}
version = latestVersion
}
local, err := b.pluginManager.InstallGitRepo(ctx, path, version)
if err != nil {
return errors.Wrap(err, "failed to get plugin path")
}
b.repositories[name] = local
}
return nil
}

66
pkg/component/pluginmanager/git.go
View file

@ -16,8 +16,15 @@ package pluginmanager
import (
"context"
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/mholt/archiver"
"github.com/storyicon/powerproto/pkg/util/command"
"github.com/storyicon/powerproto/pkg/util/logger"
)
@ -89,3 +96,62 @@ func ListGitTags(ctx context.Context, log logger.Logger, repo string) ([]string,
}
return tags, nil
}
// GithubArchive is github archive
type GithubArchive struct {
uri string
commit string
workspace string
}
// GetGithubArchive is used to download github archive
func GetGithubArchive(ctx context.Context, uri string, commitId string) (*GithubArchive, error) {
filename := fmt.Sprintf("%s.zip", commitId)
addr := fmt.Sprintf("%s/archive/%s", uri, filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, addr, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
}
}
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: addr,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &GithubArchive{
uri: uri,
commit: commitId,
workspace: workspace,
}, nil
}
// GetLocalDir is used to get local dir of archive
func (c *GithubArchive) GetLocalDir() string {
dir := path.Base(c.uri) + "-" + c.commit
return filepath.Join(c.workspace, dir)
}
// Clear is used to clear the workspace
func (c *GithubArchive) Clear() error {
return os.RemoveAll(c.workspace)
}

81
pkg/component/pluginmanager/googleapis.go
View file

@ -1,81 +0,0 @@
// Copyright 2021 storyicon@foxmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pluginmanager
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"github.com/mholt/archiver"
)
// GoogleAPIRelease defines the release of google api
type GoogleAPIRelease struct {
workspace string
commit string
}
// GetDir is used to get the dir of google api
func (p *GoogleAPIRelease) GetDir() string {
return filepath.Join(p.workspace, "googleapis-"+p.commit)
}
// Clear is used to clear the workspace
func (p *GoogleAPIRelease) Clear() error {
return os.RemoveAll(p.workspace)
}
// GetGoogleAPIRelease is used to get the release of google api
func GetGoogleAPIRelease(ctx context.Context, commitId string) (*GoogleAPIRelease, error) {
filename := fmt.Sprintf("%s.zip", commitId)
uri := fmt.Sprintf("https://github.com/googleapis/googleapis/archive/%s", filename)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
workspace, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
}
}
zipFilePath := filepath.Join(workspace, filename)
if err := downloadFile(resp, zipFilePath); err != nil {
return nil, &ErrHTTPDownload{
Url: uri,
Err: err,
Code: resp.StatusCode,
}
}
zip := archiver.NewZip()
if err := zip.Unarchive(zipFilePath, workspace); err != nil {
return nil, err
}
return &GoogleAPIRelease{
commit: commitId,
workspace: workspace,
}, nil
}

72
pkg/component/pluginmanager/manager.go
View file

@ -41,16 +41,17 @@ type PluginManager interface {
// InstallPlugin is used to install plugin
InstallPlugin(ctx context.Context, path string, version string) (local string, err error)
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
GetGoogleAPIsLatestVersion(ctx context.Context) (string, error)
// InstallGoogleAPIs is used to install google apis
InstallGoogleAPIs(ctx context.Context, commitId string) (local string, err error)
// ListGoogleAPIsVersions is used to list protoc version
ListGoogleAPIsVersions(ctx context.Context) ([]string, error)
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error)
// GoogleAPIsPath returns the googleapis path
GoogleAPIsPath(ctx context.Context, commitId string) string
// GetGitRepoLatestVersion is used to get the latest version of google apis
GetGitRepoLatestVersion(ctx context.Context, uri string) (string, error)
// InstallGitRepo is used to install google apis
InstallGitRepo(ctx context.Context, uri string, commitId string) (local string, err error)
// ListGitRepoVersions is used to list protoc version
ListGitRepoVersions(ctx context.Context, uri string) ([]string, error)
// IsGitRepoInstalled is used to check whether the protoc is installed
IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error)
// GitRepoPath returns the googleapis path
GitRepoPath(ctx context.Context, commitId string) string
// GetProtocLatestVersion is used to get the latest version of protoc
GetProtocLatestVersion(ctx context.Context) (string, error)
// ListProtocVersions is used to list protoc version
@ -140,9 +141,9 @@ func (b *BasicPluginManager) InstallPlugin(ctx context.Context, path string, ver
return InstallPluginUsingGo(ctx, b.Logger, b.storageDir, path, version)
}
// GetGoogleAPIsLatestVersion is used to get the latest version of google apis
func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (string, error) {
versions, err := b.ListGoogleAPIsVersions(ctx)
// GetGitRepoLatestVersion is used to get the latest version of google apis
func (b *BasicPluginManager) GetGitRepoLatestVersion(ctx context.Context, url string) (string, error) {
versions, err := b.ListGitRepoVersions(ctx, url)
if err != nil {
return "", err
}
@ -152,60 +153,67 @@ func (b *BasicPluginManager) GetGoogleAPIsLatestVersion(ctx context.Context) (st
return versions[len(versions)-1], nil
}
// InstallGoogleAPIs is used to install google apis
func (b *BasicPluginManager) InstallGoogleAPIs(ctx context.Context, commitId string) (string, error) {
// InstallGitRepo is used to install google apis
func (b *BasicPluginManager) InstallGitRepo(ctx context.Context, uri string, commitId string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
exists, local, err := b.IsGitRepoInstalled(ctx, uri, commitId)
if err != nil {
return "", err
}
if exists {
return local, nil
}
release, err := GetGoogleAPIRelease(ctx, commitId)
release, err := GetGithubArchive(ctx, uri, commitId)
if err != nil {
return "", err
}
defer release.Clear()
if err := util.CopyDirectory(release.GetDir(), local); err != nil {
codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
if err != nil {
return "", err
}
if err := util.CopyDirectory(release.GetLocalDir(), codePath); err != nil {
return "", err
}
return local, nil
}
// ListGoogleAPIsVersions is used to list protoc version
func (b *BasicPluginManager) ListGoogleAPIsVersions(ctx context.Context) ([]string, error) {
// ListGitRepoVersions is used to list protoc version
func (b *BasicPluginManager) ListGitRepoVersions(ctx context.Context, uri string) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultExecuteTimeout)
defer cancel()
b.versionsLock.RLock()
versions, ok := b.versions["googleapis"]
versions, ok := b.versions[uri]
b.versionsLock.RUnlock()
if ok {
return versions, nil
}
versions, err := ListGitCommitIds(ctx, b.Logger, consts.GoogleAPIsRepository)
versions, err := ListGitCommitIds(ctx, b.Logger, uri)
if err != nil {
return nil, err
}
b.versionsLock.Lock()
b.versions["googleapis"] = versions
b.versions[uri] = versions
b.versionsLock.Unlock()
return versions, nil
}
// IsGoogleAPIsInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsGoogleAPIsInstalled(ctx context.Context, commitId string) (bool, string, error) {
local := PathForGoogleAPIs(b.storageDir, commitId)
exists, err := util.IsDirExists(local)
return exists, local, err
// IsGitRepoInstalled is used to check whether the protoc is installed
func (b *BasicPluginManager) IsGitRepoInstalled(ctx context.Context, uri string, commitId string) (bool, string, error) {
codePath, err := PathForGitReposCode(b.storageDir, uri, commitId)
if err != nil {
return false, "", err
}
exists, err := util.IsDirExists(codePath)
return exists, PathForGitRepos(b.storageDir, commitId), err
}
// GoogleAPIsPath returns the googleapis path
func (b *BasicPluginManager) GoogleAPIsPath(ctx context.Context, commitId string) string {
return PathForGoogleAPIs(b.storageDir, commitId)
// GitRepoPath returns the googleapis path
func (b *BasicPluginManager) GitRepoPath(ctx context.Context, commitId string) string {
return PathForGitRepos(b.storageDir, commitId)
}
// IsProtocInstalled is used to check whether the protoc is installed

17
pkg/component/pluginmanager/paths.go
View file

@ -15,6 +15,7 @@
package pluginmanager
import (
"net/url"
"path"
"path/filepath"
@ -46,9 +47,19 @@ func GetPluginPath(path string, version string) (string, error) {
return filepath.Join(enc + "@" + encVer), nil
}
// PathForGoogleAPIs is used to get the google apis path
func PathForGoogleAPIs(storageDir string, commitId string) string {
return filepath.Join(storageDir, "googleapis", commitId)
// PathForGitReposCode returns the code path for git repos
func PathForGitReposCode(storageDir string, uri string, commitId string) (string, error) {
parsed, err := url.Parse(uri)
if err != nil {
return "", err
}
dir := parsed.Host + parsed.Path
return filepath.Join(PathForGitRepos(storageDir, commitId), dir), nil
}
// PathForGitRepos is used to get the git repo local path
func PathForGitRepos(storageDir string, commitId string) string {
return filepath.Join(storageDir, "gits", commitId)
}
// PathForPluginDir is used to get the local directory where the specified version plug-in should be stored

12
pkg/component/pluginmanager/pluginmanager_test.go
View file

@ -70,20 +70,20 @@ var _ = Describe("Pluginmanager", func() {
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())
})
It("should able to install googleapis", func() {
versions, err := manager.ListGoogleAPIsVersions(context.TODO())
It("should able to install git repos", func() {
const uri = "https://github.com/gogo/protobuf"
versions, err := manager.ListGitRepoVersions(context.TODO(), uri)
Expect(err).To(BeNil())
Expect(len(versions) > 0).To(BeTrue())
latestVersion, err := manager.GetGoogleAPIsLatestVersion(context.TODO())
latestVersion, err := manager.GetGitRepoLatestVersion(context.TODO(), uri)
Expect(err).To(BeNil())
Expect(latestVersion).To(Equal(versions[len(versions)-1]))
local, err := manager.InstallGoogleAPIs(context.TODO(), latestVersion)
local, err := manager.InstallGitRepo(context.TODO(), uri, latestVersion)
Expect(err).To(BeNil())
Expect(len(local) > 0).To(BeTrue())
exists, local, err := manager.IsGoogleAPIsInstalled(context.TODO(), latestVersion)
exists, local, err := manager.IsGitRepoInstalled(context.TODO(), uri, latestVersion)
Expect(err).To(BeNil())
Expect(exists).To(BeTrue())
Expect(len(local) != 0).To(BeTrue())