This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 700cce58a refactor(gitextractor): move "clone a repo" from
"PrePareTaskData" to a new subtask called "CloneGitRepo" (#5992)
700cce58a is described below
commit 700cce58a7f78a85ed69f87bd6c1134908f45e7f
Author: Lynwee <[email protected]>
AuthorDate: Mon Sep 4 13:56:50 2023 +0800
refactor(gitextractor): move "clone a repo" from "PrePareTaskData" to a new
subtask called "CloneGitRepo" (#5992)
---
backend/plugins/gitextractor/impl/impl.go | 34 ++--------
backend/plugins/gitextractor/main.go | 2 +-
.../gitextractor/{impl/impl.go => tasks/clone.go} | 76 ++++++----------------
.../gitextractor/tasks/git_repo_collector.go | 18 ++++-
4 files changed, 40 insertions(+), 90 deletions(-)
diff --git a/backend/plugins/gitextractor/impl/impl.go
b/backend/plugins/gitextractor/impl/impl.go
index 81fe8e134..b42c5d5bb 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -18,18 +18,12 @@ limitations under the License.
package impl
import (
- "context"
- "fmt"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitextractor/models"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
- "github.com/apache/incubator-devlake/plugins/gitextractor/store"
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
- "strings"
)
var _ interface {
@@ -55,6 +49,7 @@ func (p GitExtractor) Name() string {
// return all available subtasks, framework will run them for you in order
func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
+ tasks.CloneGitRepoMeta,
tasks.CollectGitCommitMeta,
tasks.CollectGitBranchMeta,
tasks.CollectGitTagMeta,
@@ -62,7 +57,7 @@ func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
}
}
-// based on task context and user input options, return data that shared among
all subtasks
+// PrepareTaskData based on task context and user input options, return data
that shared among all subtasks
func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
var op tasks.GitExtractorOptions
if err := helper.Decode(options, &op, nil); err != nil {
@@ -71,12 +66,10 @@ func (p GitExtractor) PrepareTaskData(taskCtx
plugin.TaskContext, options map[st
if err := op.Valid(); err != nil {
return nil, err
}
- storage := store.NewDatabase(taskCtx, op.RepoId)
- repo, err := NewGitRepo(taskCtx.GetContext(), taskCtx.GetLogger(),
storage, op)
- if err != nil {
- return nil, err
+ taskData := &tasks.GitExtractorTaskData{
+ Options: &op,
}
- return repo, nil
+ return taskData, nil
}
func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
@@ -91,20 +84,3 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext)
errors.Error {
func (p GitExtractor) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/gitextractor"
}
-
-// NewGitRepo create and return a new parser git repo
-func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store,
op tasks.GitExtractorOptions) (*parser.GitRepo, errors.Error) {
- var err errors.Error
- var repo *parser.GitRepo
- p := parser.NewGitRepoCreator(storage, logger)
- if strings.HasPrefix(op.Url, "http") {
- repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User,
op.Password, op.Proxy)
- } else if url := strings.TrimPrefix(op.Url, "ssh://");
strings.HasPrefix(url, "git@") {
- repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey,
op.Passphrase)
- } else if strings.HasPrefix(op.Url, "/") {
- repo, err = p.LocalRepo(op.Url, op.RepoId)
- } else {
- return nil, errors.BadInput.New(fmt.Sprintf("unsupported url
[%s]", op.Url))
- }
- return repo, err
-}
diff --git a/backend/plugins/gitextractor/main.go
b/backend/plugins/gitextractor/main.go
index f1fb468d2..0224b0c97 100644
--- a/backend/plugins/gitextractor/main.go
+++ b/backend/plugins/gitextractor/main.go
@@ -78,7 +78,7 @@ func main() {
"git extractor",
nil,
)
- repo, err := impl.NewGitRepo(subTaskCtx.GetContext(), logger, storage,
tasks.GitExtractorOptions{
+ repo, err := tasks.NewGitRepo(subTaskCtx.GetContext(), logger, storage,
&tasks.GitExtractorOptions{
RepoId: *id,
Url: *url,
User: *user,
diff --git a/backend/plugins/gitextractor/impl/impl.go
b/backend/plugins/gitextractor/tasks/clone.go
similarity index 51%
copy from backend/plugins/gitextractor/impl/impl.go
copy to backend/plugins/gitextractor/tasks/clone.go
index 81fe8e134..88c769403 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/tasks/clone.go
@@ -15,85 +15,47 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package impl
+package tasks
import (
"context"
"fmt"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
- "github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
"strings"
)
-var _ interface {
- plugin.PluginMeta
- plugin.PluginTask
- plugin.PluginModel
-} = (*GitExtractor)(nil)
-
-type GitExtractor struct{}
-
-func (p GitExtractor) GetTablesInfo() []dal.Tabler {
- return []dal.Tabler{}
-}
-
-func (p GitExtractor) Description() string {
- return "extract infos from git repository"
+var CloneGitRepoMeta = plugin.SubTaskMeta{
+ Name: "cloneGitRepo",
+ EntryPoint: CloneGitRepo,
+ EnabledByDefault: true,
+ Required: true,
+ Description: "clone a git repo, make it available to later tasks",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
}
-func (p GitExtractor) Name() string {
- return "gitextractor"
-}
-
-// return all available subtasks, framework will run them for you in order
-func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
- return []plugin.SubTaskMeta{
- tasks.CollectGitCommitMeta,
- tasks.CollectGitBranchMeta,
- tasks.CollectGitTagMeta,
- tasks.CollectGitDiffLineMeta,
+func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
+ taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
+ if !ok {
+ panic("git repo reference not found on context")
}
-}
-
-// based on task context and user input options, return data that shared among
all subtasks
-func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
- var op tasks.GitExtractorOptions
- if err := helper.Decode(options, &op, nil); err != nil {
- return nil, err
- }
- if err := op.Valid(); err != nil {
- return nil, err
- }
- storage := store.NewDatabase(taskCtx, op.RepoId)
- repo, err := NewGitRepo(taskCtx.GetContext(), taskCtx.GetLogger(),
storage, op)
+ op := taskData.Options
+ storage := store.NewDatabase(subTaskCtx, op.RepoId)
+ repo, err := NewGitRepo(subTaskCtx.GetContext(),
subTaskCtx.GetLogger(), storage, op)
if err != nil {
- return nil, err
- }
- return repo, nil
-}
-
-func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
- if repo, ok := taskCtx.GetData().(*parser.GitRepo); ok {
- if err := repo.Close(); err != nil {
- return errors.Convert(err)
- }
+ return err
}
+ taskData.GitRepo = repo
+ subTaskCtx.TaskContext().SetData(taskData)
return nil
}
-func (p GitExtractor) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/gitextractor"
-}
-
// NewGitRepo create and return a new parser git repo
-func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store,
op tasks.GitExtractorOptions) (*parser.GitRepo, errors.Error) {
+func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store,
op *GitExtractorOptions) (*parser.GitRepo, errors.Error) {
var err errors.Error
var repo *parser.GitRepo
p := parser.NewGitRepoCreator(storage, logger)
diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go
b/backend/plugins/gitextractor/tasks/git_repo_collector.go
index 45090f760..d5132fa6c 100644
--- a/backend/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/backend/plugins/gitextractor/tasks/git_repo_collector.go
@@ -25,9 +25,14 @@ import (
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
)
+type GitExtractorTaskData struct {
+ Options *GitExtractorOptions
+ GitRepo *parser.GitRepo
+}
+
type GitExtractorOptions struct {
RepoId string `json:"repoId"`
- Name string `jsno:"name"`
+ Name string `json:"name"`
Url string `json:"url"`
User string `json:"user"`
Password string `json:"password"`
@@ -99,11 +104,14 @@ func CollectGitDiffLines(subTaskCtx plugin.SubTaskContext)
errors.Error {
}
func getGitRepo(subTaskCtx plugin.SubTaskContext) *parser.GitRepo {
- repo, ok := subTaskCtx.GetData().(*parser.GitRepo)
+ taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
if !ok {
panic("git repo reference not found on context")
}
- return repo
+ if taskData.GitRepo == nil {
+ panic("git repo is empty, please check subtask: clone repo")
+ }
+ return taskData.GitRepo
}
var CollectGitCommitMeta = plugin.SubTaskMeta{
@@ -112,6 +120,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "collect git commits into Domain Layer Tables",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE,
plugin.DOMAIN_TYPE_CROSS},
+ Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
}
var CollectGitBranchMeta = plugin.SubTaskMeta{
@@ -120,6 +129,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "collect git branch into Domain Layer Tables",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
}
var CollectGitTagMeta = plugin.SubTaskMeta{
@@ -128,6 +138,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "collect git tag into Domain Layer Tables",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
}
var CollectGitDiffLineMeta = plugin.SubTaskMeta{
@@ -136,4 +147,5 @@ var CollectGitDiffLineMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "collect git commit diff line into Domain Layer
Tables",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
}
