This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 6f57a15a9 fix: git clone failing on azuredevops (#7356)
6f57a15a9 is described below
commit 6f57a15a90691705f73a8ede71a3355d7cbaa120
Author: Klesh Wong <[email protected]>
AuthorDate: Mon Apr 22 15:24:52 2024 +0800
fix: git clone failing on azuredevops (#7356)
* fix: git clone failing on azuredevops
* fix: unit test
---
.../plugins/azuredevops_go/api/blueprint_v200.go | 9 +++--
.../azuredevops_go/api/blueprint_v200_test.go | 9 +++--
.../plugins/gitextractor/parser/clone_gitcli.go | 43 +++++++++++++---------
backend/plugins/gitextractor/parser/taskdata.go | 3 +-
.../python/plugins/azuredevops/azuredevops/main.py | 2 +-
.../python/pydevlake/pydevlake/pipeline_tasks.py | 5 ++-
6 files changed, 41 insertions(+), 30 deletions(-)
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go
b/backend/plugins/azuredevops_go/api/blueprint_v200.go
index a9d178e47..f1df9be8c 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go
@@ -136,10 +136,11 @@ func makePipelinePlanV200(
stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "name": azuredevopsRepo.Name,
- "repoId":
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID,
azuredevopsRepo.Id),
- "proxy": connection.Proxy,
+ "url": cloneUrl.String(),
+ "name": azuredevopsRepo.Name,
+ "repoId":
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID,
azuredevopsRepo.Id),
+ "proxy": connection.Proxy,
+ "noShallowClone": true,
},
})
}
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
index eeba43c18..8fb55c893 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
@@ -153,10 +153,11 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "proxy": "",
- "repoId": expectDomainScopeId,
- "name": azureDevOpsProjectName,
- "url":
"https://git:personal-access-token@this_is_cloneUrl";,
+ "proxy": "",
+ "repoId": expectDomainScopeId,
+ "name":
azureDevOpsProjectName,
+ "url":
"https://git:personal-access-token@this_is_cloneUrl";,
+ "noShallowClone": true,
},
},
},
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go
b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 500a30c97..1a666ad42 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -50,38 +50,45 @@ func NewGitcliCloner(basicRes context.BasicRes)
*GitcliCloner {
func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string)
errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
- // load state
- stateManager, err := api.NewCollectorStateManager(
- ctx,
- ctx.TaskContext().SyncPolicy(),
- "gitextractor",
- fmt.Sprintf(
- `{"RepoId: "%s","SkipCommitStat": %v,
"SkipCommitFiles": %v}`,
- taskData.Options.RepoId,
- *taskData.Options.SkipCommitStat,
- *taskData.Options.SkipCommitFiles,
- ),
- )
- if err != nil {
- return err
+ var since *time.Time
+ if !taskData.Options.NoShallowClone {
+ // load state
+ stateManager, err := api.NewCollectorStateManager(
+ ctx,
+ ctx.TaskContext().SyncPolicy(),
+ "gitextractor",
+ fmt.Sprintf(
+ `{"RepoId: "%s","SkipCommitStat": %v,
"SkipCommitFiles": %v}`,
+ taskData.Options.RepoId,
+ *taskData.Options.SkipCommitStat,
+ *taskData.Options.SkipCommitFiles,
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ g.stateManager = stateManager
+ since = stateManager.GetSince()
}
- g.stateManager = stateManager
- cmd, err := g.buildCloneCommand(ctx, localDir,
g.stateManager.GetSince())
+ cmd, err := g.buildCloneCommand(ctx, localDir, since)
if err != nil {
return err
}
err = g.execCloneCommand(cmd)
if err != nil {
// it is likely that nothing to collect on incrmental mode
- if errors.Is(err, ErrShallowInfoProcessing) &&
stateManager.IsIncremental() {
+ if errors.Is(err, ErrShallowInfoProcessing) && g.stateManager
!= nil && g.stateManager.IsIncremental() {
return ErrNoDataOnIncrementalMode
}
return err
}
// save state
- return g.stateManager.Close()
+ if g.stateManager != nil {
+ return g.stateManager.Close()
+ }
+ return nil
}
func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir
string, since *time.Time) (*exec.Cmd, errors.Error) {
diff --git a/backend/plugins/gitextractor/parser/taskdata.go
b/backend/plugins/gitextractor/parser/taskdata.go
index b0a44c660..a2d894d02 100644
--- a/backend/plugins/gitextractor/parser/taskdata.go
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -25,7 +25,7 @@ type GitExtractorTaskData struct {
Options *GitExtractorOptions
ParsedURL *url.URL
GitRepo RepoCollector
- SkipAllSubtasks bool // skip all tasks without error if true
+ SkipAllSubtasks bool // siliently skip all tasks without raising error
}
type GitExtractorOptions struct {
@@ -40,4 +40,5 @@ type GitExtractorOptions struct {
UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"`
SkipCommitStat *bool `json:"skipCommitStat"
mapstructure:"skipCommitStat" comment:"skip all commit stat including
added/deleted lines and commit files as well"`
SkipCommitFiles *bool `json:"skipCommitFiles"
mapstructure:"skipCommitFiles"`
+ NoShallowClone bool `json:"noShallowClone"
mapstructure:"noShallowClone"`
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index 2560ab439..e66422522 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -135,7 +135,7 @@ class AzureDevOpsPlugin(Plugin):
if DomainType.CODE in scope_config.domain_types and not
scope.is_external():
url = urlparse(scope.remote_url)
url =
url._replace(netloc=f'{url.username}:{connection.token.get_secret_value()}@{url.hostname}')
- yield gitextractor(url.geturl(), scope.name, scope.domain_id(),
connection.proxy)
+ yield gitextractor(url.geturl(), scope.name, scope.domain_id(),
connection.proxy, True)
def extra_stages(self, scope_config_pairs: list[tuple[GitRepository,
GitRepositoryConfig]], _):
for scope, config in scope_config_pairs:
diff --git a/backend/python/pydevlake/pydevlake/pipeline_tasks.py
b/backend/python/pydevlake/pydevlake/pipeline_tasks.py
index b62aaeff6..751d0069f 100644
--- a/backend/python/pydevlake/pydevlake/pipeline_tasks.py
+++ b/backend/python/pydevlake/pydevlake/pipeline_tasks.py
@@ -21,7 +21,7 @@ from pydantic import BaseModel
from pydevlake.message import PipelineTask
-def gitextractor(url: str, repo_name: str, scope_id: str, proxy:
Optional[str]):
+def gitextractor(url: str, repo_name: str, scope_id: str, proxy:
Optional[str], no_shallow_clone: Optional[bool] = False):
return PipelineTask(
plugin="gitextractor",
options={
@@ -29,7 +29,8 @@ def gitextractor(url: str, repo_name: str, scope_id: str,
proxy: Optional[str]):
"name": repo_name,
"fullName": repo_name,
"repoId": scope_id,
- "proxy": proxy
+ "proxy": proxy,
+ "noShallowClone": no_shallow_clone,
},
)
