This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3035e057a fix(github): adapt bp v100 to v200 (#3922)
3035e057a is described below
commit 3035e057a6f8d78169ad3fc026d32092628556b3
Author: Warren Chen <[email protected]>
AuthorDate: Tue Dec 13 21:36:57 2022 +0800
fix(github): adapt bp v100 to v200 (#3922)
---
models/domainlayer/devops/cicd_pipeline_commmit.go | 2 +-
plugins/bitbucket/tasks/pipeline_convertor.go | 2 +-
.../dora/e2e/raw_tables/cicd_pipeline_commits.csv | 2 +-
plugins/github/api/blueprint.go | 18 +--
plugins/github/api/blueprint_V200_test.go | 2 -
plugins/github/api/blueprint_test.go | 4 +-
plugins/github/api/blueprint_v200.go | 2 +-
plugins/github/api/scope.go | 3 +-
plugins/github/e2e/account_test.go | 5 +-
plugins/github/e2e/cicd_test.go | 4 +-
plugins/github/e2e/comment_test.go | 5 +-
plugins/github/e2e/event_test.go | 5 +-
plugins/github/e2e/issue_test.go | 5 +-
plugins/github/e2e/milestone_test.go | 15 +--
plugins/github/e2e/pr_commit_test.go | 5 +-
plugins/github/e2e/pr_enrich_issue_test.go | 5 +-
plugins/github/e2e/pr_review_test.go | 5 +-
plugins/github/e2e/pr_test.go | 5 +-
plugins/github/e2e/repo_test.go | 5 +-
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 42 +++---
plugins/github/impl/impl.go | 90 ++++++++++++-
plugins/github/models/repo.go | 2 +-
plugins/github/tasks/account_collector.go | 2 +-
plugins/github/tasks/account_convertor.go | 2 +-
plugins/github/tasks/account_org_collector.go | 2 +-
plugins/github/tasks/cicd_job_collector.go | 2 +-
plugins/github/tasks/cicd_job_convertor.go | 2 +-
plugins/github/tasks/cicd_job_extractor.go | 2 +-
plugins/github/tasks/cicd_run_convertor.go | 2 +-
plugins/github/tasks/cicd_run_extractor.go | 2 +-
plugins/github/tasks/comment_extractor.go | 6 +-
plugins/github/tasks/commit_convertor.go | 2 +-
plugins/github/tasks/commit_extractor.go | 2 +-
plugins/github/tasks/commit_stats_collector.go | 4 +-
plugins/github/tasks/event_extractor.go | 2 +-
plugins/github/tasks/issue_comment_convertor.go | 2 +-
plugins/github/tasks/issue_convertor.go | 2 +-
plugins/github/tasks/issue_extractor.go | 6 +-
plugins/github/tasks/issue_label_convertor.go | 2 +-
plugins/github/tasks/milestone_converter.go | 2 +-
plugins/github/tasks/milestone_extractor.go | 2 +-
plugins/github/tasks/pr_comment_convertor.go | 2 +-
plugins/github/tasks/pr_commit_collector.go | 2 +-
plugins/github/tasks/pr_commit_convertor.go | 2 +-
plugins/github/tasks/pr_commit_extractor.go | 2 +-
plugins/github/tasks/pr_convertor.go | 2 +-
plugins/github/tasks/pr_extractor.go | 4 +-
plugins/github/tasks/pr_issue_convertor.go | 2 +-
plugins/github/tasks/pr_issue_enricher.go | 2 +-
plugins/github/tasks/pr_label_convertor.go | 2 +-
plugins/github/tasks/pr_review_collector.go | 2 +-
.../github/tasks/pr_review_comment_extractor.go | 4 +-
plugins/github/tasks/pr_review_convertor.go | 2 +-
plugins/github/tasks/pr_review_extractor.go | 2 +-
plugins/github/tasks/repo_convertor.go | 15 ++-
plugins/github/tasks/task_data.go | 2 +-
plugins/github_graphql/impl/impl.go | 59 +--------
plugins/github_graphql/tasks/account_collector.go | 2 +-
.../github_graphql/tasks/check_run_collector.go | 4 +-
plugins/github_graphql/tasks/issue_collector.go | 8 +-
plugins/github_graphql/tasks/pr_collector.go | 8 +-
plugins/github_graphql/tasks/repo_collector.go | 142 ---------------------
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
plugins/jenkins/e2e/builds_test.go | 2 +-
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
plugins/jenkins/tasks/build_commit_convertor.go | 2 +-
.../e2e/raw_tables/cicd_pipeline_commits.csv | 2 +-
67 files changed, 206 insertions(+), 355 deletions(-)
diff --git a/models/domainlayer/devops/cicd_pipeline_commmit.go
b/models/domainlayer/devops/cicd_pipeline_commmit.go
index 68236f392..df00ef692 100644
--- a/models/domainlayer/devops/cicd_pipeline_commmit.go
+++ b/models/domainlayer/devops/cicd_pipeline_commmit.go
@@ -25,7 +25,7 @@ type CiCDPipelineCommit struct {
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
Branch string `gorm:"type:varchar(255)"`
RepoId string `gorm:"index;type:varchar(255)"`
- RepoUrl string
+ Repo string
}
func (CiCDPipelineCommit) TableName() string {
diff --git a/plugins/bitbucket/tasks/pipeline_convertor.go
b/plugins/bitbucket/tasks/pipeline_convertor.go
index 68c9f3c6b..7b97916c6 100644
--- a/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -77,7 +77,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext)
errors.Error {
Generate(bitbucketPipeline.ConnectionId, bitbucketPipeline.RepoId),
CommitSha: bitbucketPipeline.CommitSha,
Branch: bitbucketPipeline.RefName,
- RepoUrl: bitbucketPipeline.WebUrl,
+ Repo: bitbucketPipeline.WebUrl,
}
domainPipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
diff --git a/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
b/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
index 3c5029696..d66671474 100644
--- a/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
+++ b/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url
+pipeline_id,commit_sha,branch,repo_id,repo
pipeline110,commit300,z5z07j8qDO,repo1,
pipeline111,commit301,7j3t5IFWb6,repo1,
pipeline112,commit302,VBSqoF8WIh,repo1,
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 55622d218..891016de5 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -43,14 +43,10 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta,
connectionId uint64, scop
if err != nil {
return nil, err
}
- token := strings.Split(connection.Token, ",")[0]
-
apiClient, err := helper.NewApiClient(
context.TODO(),
connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %s", token),
- },
+ nil,
10*time.Second,
connection.Proxy,
basicRes,
@@ -65,11 +61,11 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta,
connectionId uint64, scop
return plan, nil
}
-func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope
[]*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection
*models.GithubConnection) (core.PipelinePlan, errors.Error) {
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scopeV100s
[]*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection
*models.GithubConnection) (core.PipelinePlan, errors.Error) {
var err errors.Error
var repo *tasks.GithubApiRepo
- plan := make(core.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
+ plan := make(core.PipelinePlan, len(scopeV100s))
+ for i, scopeElem := range scopeV100s {
// handle taskOptions and transformationRules, by dumping them
to taskOptions
transformationRules := make(map[string]interface{})
if len(scopeElem.Transformation) > 0 {
@@ -118,7 +114,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
return nil, err
}
// collect git data by gitextractor if CODE was requested
- repo, err = memorizedGetApiRepo(repo, op, apiClient)
+ repo, err = MemorizedGetApiRepo(repo, op, apiClient)
if err != nil {
return nil, err
}
@@ -139,7 +135,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
if j == len(plan) {
plan = append(plan, nil)
}
- repo, err = memorizedGetApiRepo(repo, op, apiClient)
+ repo, err = MemorizedGetApiRepo(repo, op, apiClient)
if err != nil {
return nil, err
}
@@ -246,7 +242,7 @@ func getApiRepo(op *tasks.GithubOptions, apiClient
helper.ApiClientGetter) (*tas
return repoRes, nil
}
-func memorizedGetApiRepo(repo *tasks.GithubApiRepo, op *tasks.GithubOptions,
apiClient helper.ApiClientGetter) (*tasks.GithubApiRepo, errors.Error) {
+func MemorizedGetApiRepo(repo *tasks.GithubApiRepo, op *tasks.GithubOptions,
apiClient helper.ApiClientGetter) (*tasks.GithubApiRepo, errors.Error) {
if repo == nil {
var err errors.Error
repo, err = getApiRepo(op, apiClient)
diff --git a/plugins/github/api/blueprint_V200_test.go
b/plugins/github/api/blueprint_V200_test.go
index 07e3b3198..5f32660e6 100644
--- a/plugins/github/api/blueprint_V200_test.go
+++ b/plugins/github/api/blueprint_V200_test.go
@@ -34,7 +34,6 @@ import (
"io"
"net/http"
"testing"
- "time"
)
func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
@@ -147,7 +146,6 @@ func NewMockBasicRes() *mocks.BasicRes {
GithubId: 12345,
Name: "test/testRepo",
TransformationRuleId: 1,
- CreatedDate: time.Time{},
}
testTransformationRule := &models.GithubTransformationRule{
diff --git a/plugins/github/api/blueprint_test.go
b/plugins/github/api/blueprint_test.go
index 54ae4cbd2..e83332835 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -132,11 +132,11 @@ func TestMakePipelinePlan(t *testing.T) {
func TestMemorizedGetApiRepo(t *testing.T) {
op := prepareOptions(t)
expect := repo
- repo1, err := memorizedGetApiRepo(repo, op, nil)
+ repo1, err := MemorizedGetApiRepo(repo, op, nil)
assert.Nil(t, err)
assert.Equal(t, expect, repo1)
mockApiClient := prepareMockClient(t, repo)
- repo2, err := memorizedGetApiRepo(nil, op, mockApiClient)
+ repo2, err := MemorizedGetApiRepo(nil, op, mockApiClient)
assert.Nil(t, err)
assert.NotEqual(t, expect, repo2)
}
diff --git a/plugins/github/api/blueprint_v200.go
b/plugins/github/api/blueprint_v200.go
index 71e8e25a5..d42fc3078 100644
--- a/plugins/github/api/blueprint_v200.go
+++ b/plugins/github/api/blueprint_v200.go
@@ -148,7 +148,7 @@ func makeDataSourcePipelinePlanV200(
op.Repo = ownerRepo[1]
// add gitex stage and add repo to scopes
if utils.StringsContains(bpScope.Entities,
core.DOMAIN_TYPE_CODE) {
- repoRes, err = memorizedGetApiRepo(repoRes, op,
apiClient)
+ repoRes, err = MemorizedGetApiRepo(repoRes, op,
apiClient)
if err != nil {
return nil, err
}
diff --git a/plugins/github/api/scope.go b/plugins/github/api/scope.go
index 29e27ac9e..008780188 100644
--- a/plugins/github/api/scope.go
+++ b/plugins/github/api/scope.go
@@ -70,7 +70,8 @@ func PutScope(input *core.ApiResourceInput)
(*core.ApiResourceOutput, errors.Err
keeper[repo.GithubId] = struct{}{}
}
repo.ConnectionId = connectionId
- repo.CreatedDate = now
+ // Fixme: why do we set this to now?
+ repo.CreatedDate = &now
repo.UpdatedDate = &now
err = verifyRepo(repo)
if err != nil {
diff --git a/plugins/github/e2e/account_test.go
b/plugins/github/e2e/account_test.go
index cb778b0d0..458b04083 100644
--- a/plugins/github/e2e/account_test.go
+++ b/plugins/github/e2e/account_test.go
@@ -31,16 +31,13 @@ func TestAccountDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/cicd_test.go b/plugins/github/e2e/cicd_test.go
index e1bb4a215..7fc836619 100644
--- a/plugins/github/e2e/cicd_test.go
+++ b/plugins/github/e2e/cicd_test.go
@@ -37,14 +37,12 @@ func TestGithubCICDDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
DeploymentPattern: `deploy.*`,
ProductionPattern: `deploywindows.*`,
},
},
- Repo: &models.GithubRepo{
- GithubId: 134018330,
- },
}
// import raw data table
diff --git a/plugins/github/e2e/comment_test.go
b/plugins/github/e2e/comment_test.go
index a082606cc..5e364f11f 100644
--- a/plugins/github/e2e/comment_test.go
+++ b/plugins/github/e2e/comment_test.go
@@ -32,14 +32,12 @@ func TestCommentDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
@@ -52,7 +50,6 @@ func TestCommentDataFlow(t *testing.T) {
IssueTypeRequirement:
"^(feat|feature|proposal|requirement)$",
},
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/event_test.go b/plugins/github/e2e/event_test.go
index 04054a61c..27a1c6a09 100644
--- a/plugins/github/e2e/event_test.go
+++ b/plugins/github/e2e/event_test.go
@@ -31,16 +31,13 @@ func TestEventDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/issue_test.go b/plugins/github/e2e/issue_test.go
index 9ee209dba..60678b704 100644
--- a/plugins/github/e2e/issue_test.go
+++ b/plugins/github/e2e/issue_test.go
@@ -31,14 +31,12 @@ func TestIssueDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
@@ -51,7 +49,6 @@ func TestIssueDataFlow(t *testing.T) {
IssueTypeRequirement:
"^(feat|feature|proposal|requirement)$",
},
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/milestone_test.go
b/plugins/github/e2e/milestone_test.go
index a22c1dbe7..a6f75d45c 100644
--- a/plugins/github/e2e/milestone_test.go
+++ b/plugins/github/e2e/milestone_test.go
@@ -18,32 +18,24 @@ limitations under the License.
package e2e
import (
- "testing"
- "time"
-
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/github/impl"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
+ "testing"
)
func TestMilestoneDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- ConnectionId: 1,
- GithubId: 134018330,
- CreatedDate: func() time.Time {
- createdTime, _ := time.Parse(time.RFC3339,
"2006-01-02T15:04:05Z")
- return createdTime
- }(),
- }
+
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
@@ -56,7 +48,6 @@ func TestMilestoneDataFlow(t *testing.T) {
IssueTypeRequirement:
"^(feat|feature|proposal|requirement)$",
},
},
- Repo: githubRepository,
}
dataflowTester.FlushTabler(&models.GithubMilestone{})
diff --git a/plugins/github/e2e/pr_commit_test.go
b/plugins/github/e2e/pr_commit_test.go
index 9a34d84ea..744d43d18 100644
--- a/plugins/github/e2e/pr_commit_test.go
+++ b/plugins/github/e2e/pr_commit_test.go
@@ -31,16 +31,13 @@ func TestPrCommitDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/pr_enrich_issue_test.go
b/plugins/github/e2e/pr_enrich_issue_test.go
index 7de38d337..49737cbd3 100644
--- a/plugins/github/e2e/pr_enrich_issue_test.go
+++ b/plugins/github/e2e/pr_enrich_issue_test.go
@@ -31,14 +31,12 @@ func TestPrEnrichIssueDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
@@ -51,7 +49,6 @@ func TestPrEnrichIssueDataFlow(t *testing.T) {
IssueTypeRequirement:
"^(feat|feature|proposal|requirement)$",
},
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/pr_review_test.go
b/plugins/github/e2e/pr_review_test.go
index ac5886591..082228f6b 100644
--- a/plugins/github/e2e/pr_review_test.go
+++ b/plugins/github/e2e/pr_review_test.go
@@ -30,16 +30,13 @@ func TestPrReviewDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/pr_test.go b/plugins/github/e2e/pr_test.go
index 12e6f2f44..46f4404ec 100644
--- a/plugins/github/e2e/pr_test.go
+++ b/plugins/github/e2e/pr_test.go
@@ -31,21 +31,18 @@ func TestPrDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern:
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
},
},
- Repo: githubRepository,
}
// import raw data table
diff --git a/plugins/github/e2e/repo_test.go b/plugins/github/e2e/repo_test.go
index 05d459d58..7a0bf1fde 100644
--- a/plugins/github/e2e/repo_test.go
+++ b/plugins/github/e2e/repo_test.go
@@ -35,20 +35,17 @@ func TestRepoDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
- githubRepository := &models.GithubRepo{
- GithubId: 134018330,
- }
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
+ GithubId: 134018330,
GithubTransformationRule:
&models.GithubTransformationRule{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
},
},
- Repo: githubRepository,
}
// verify convert
diff --git a/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 3d8744049..e2d80cf1b 100644
--- a/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,21 +1,21 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-github:GithubRun:1:134018330:2559400712,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,20,
-github:GithubRun:1:134018330:2559400713,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,19,
-github:GithubRun:1:134018330:2559400714,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,18,
-github:GithubRun:1:134018330:2559507315,f85611741eb1f5451697ac589008d28f240887fc,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,17,
-github:GithubRun:1:134018330:2566218975,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,16,
-github:GithubRun:1:134018330:2566218976,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,15,
-github:GithubRun:1:134018330:2566218977,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,14,
-github:GithubRun:1:134018330:2589885628,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,13,
-github:GithubRun:1:134018330:2589885635,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,12,
-github:GithubRun:1:134018330:2589885639,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,11,
-github:GithubRun:1:134018330:2600408985,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,10,
-github:GithubRun:1:134018330:2639945362,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,9,
-github:GithubRun:1:134018330:2680721264,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,8,
-github:GithubRun:1:134018330:2722539966,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,7,
-github:GithubRun:1:134018330:2764660507,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,6,
-github:GithubRun:1:134018330:2807709308,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,5,
-github:GithubRun:1:134018330:2850801364,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,4,
-github:GithubRun:1:134018330:2893573709,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,3,
-github:GithubRun:1:134018330:2938072864,06e6934c35c336b1a2bd3005fb21dc3914a45747,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,2,
-github:GithubRun:1:134018330:2983238245,06e6934c35c336b1a2bd3005fb21dc3914a45747,master,github:GithubRepo:1:134018330,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_runs,1,
+pipeline_id,commit_sha,branch,repo_id,repo
+github:GithubRun:1:134018330:2559400712,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2559400713,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2559400714,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2559507315,f85611741eb1f5451697ac589008d28f240887fc,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2566218975,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2566218976,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2566218977,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2589885628,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2589885635,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2589885639,fa938334e73faf88a15b59622ab1da61a643c5da,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2600408985,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2639945362,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2680721264,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2722539966,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2764660507,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2807709308,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2850801364,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2893573709,32664cb1408f8d9ffa7236335025a4cd94a306ce,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2938072864,06e6934c35c336b1a2bd3005fb21dc3914a45747,master,github:GithubRepo:1:134018330,
+github:GithubRun:1:134018330:2983238245,06e6934c35c336b1a2bd3005fb21dc3914a45747,master,github:GithubRepo:1:134018330,
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index cc0408472..b254990df 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -19,6 +19,8 @@ package impl
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "strings"
"time"
"github.com/apache/incubator-devlake/errors"
@@ -27,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/github/tasks"
- "github.com/apache/incubator-devlake/plugins/github_graphql/impl"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/spf13/viper"
"gorm.io/gorm"
@@ -157,7 +158,11 @@ func (plugin Github) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get github
connection by the given connection ID")
}
- githubRepo, err := impl.EnrichOptions(taskCtx, op, connection)
+ apiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get github API
client instance")
+ }
+ err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
if err != nil {
return nil, err
}
@@ -169,14 +174,9 @@ func (plugin Github) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
}
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get github API
client instance")
- }
taskData := &tasks.GithubTaskData{
Options: op,
ApiClient: apiClient,
- Repo: githubRepo,
}
if !createdDateAfter.IsZero() {
@@ -246,3 +246,79 @@ func (plugin Github) Close(taskCtx core.TaskContext)
errors.Error {
data.ApiClient.Release()
return nil
}
+
+func EnrichOptions(taskCtx core.TaskContext,
+ op *tasks.GithubOptions,
+ apiClient *helper.ApiClient) errors.Error {
+ var githubRepo models.GithubRepo
+ var err errors.Error
+ log := taskCtx.GetLogger()
+ // for advanced mode or others which we already set value to onwer/repo
+ if op.Owner != "" && op.Repo != "" {
+ // Lets try to use owner_login/name to find the record first
+ // In our db._tool_github_repos, record might be two kinds:
+ // 1. owner_login = op.owner and name = op.repo
+ // 2. name = op.owner/op.repo
+ err := taskCtx.GetDal().First(&githubRepo, dal.Where(
+ "connection_id = ? AND ((name = ? AND owner_login = ?)
OR name = ?)",
+ op.ConnectionId, op.Repo, op.Owner,
fmt.Sprintf("%s/%s", op.Owner, op.Repo)))
+ if err == nil {
+ op.GithubId = githubRepo.GithubId
+ op.TransformationRuleId =
githubRepo.TransformationRuleId
+ }
+ // If we still cannot find the record in db, we have to request
from remote server and save it to db
+ if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
+ var repo *tasks.GithubApiRepo
+ repo, err = api.MemorizedGetApiRepo(repo, op, apiClient)
+ if err != nil {
+ return err
+ }
+ log.Debug(fmt.Sprintf("Current repo: %s",
repo.FullName))
+ var scope models.GithubRepo
+ scope.ConnectionId = op.ConnectionId
+ scope.GithubId = repo.GithubId
+ scope.CreatedDate = repo.CreatedAt.ToNullableTime()
+ scope.Language = repo.Language
+ scope.Description = repo.Description
+ scope.HTMLUrl = repo.HTMLUrl
+ scope.ConnectionId = op.ConnectionId
+ scope.Name = repo.FullName
+ err = taskCtx.GetDal().CreateIfNotExist(&scope)
+ if err != nil {
+ return err
+ }
+
+ op.GithubId = repo.GithubId
+ }
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("fail to
find repo %s/%s", op.Owner, op.Repo))
+ }
+ }
+ // for bp v200 which we only set ScopeId for options
+ if githubRepo.GithubId == 0 && op.ScopeId != "" {
+ log.Debug(fmt.Sprintf("Getting githubRepo by op.ScopeId: %s",
op.ScopeId))
+ err = taskCtx.GetDal().First(&githubRepo,
dal.Where(`connection_id = ? AND github_id = ?`, op.ConnectionId, op.ScopeId))
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("fail to
find repo %s", op.ScopeId))
+ }
+ ownerName := strings.Split(githubRepo.Name, "/")
+ if len(ownerName) != 2 {
+ return errors.Default.New("Fail to set owner/repo for
github options.")
+ }
+ op.Owner = ownerName[0]
+ op.Repo = ownerName[1]
+ op.GithubId = githubRepo.GithubId
+ op.TransformationRuleId = githubRepo.TransformationRuleId
+ }
+ // Set GithubTransformationRule if it's nil, this has lower priority
+ if op.GithubTransformationRule == nil && op.TransformationRuleId != 0 {
+ var transformationRule models.GithubTransformationRule
+ err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", githubRepo.TransformationRuleId))
+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.BadInput.Wrap(err, "fail to get
transformationRule")
+ }
+ op.GithubTransformationRule = &transformationRule
+ }
+ err = tasks.ValidateTaskOptions(op)
+ return err
+}
diff --git a/plugins/github/models/repo.go b/plugins/github/models/repo.go
index 8f20776b9..5faec687b 100644
--- a/plugins/github/models/repo.go
+++ b/plugins/github/models/repo.go
@@ -34,7 +34,7 @@ type GithubRepo struct {
Language string `json:"language"
gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
ParentGithubId int `json:"parentId"
mapstructure:"parentGithubId,omitempty"`
ParentHTMLUrl string `json:"parentHtmlUrl"
mapstructure:"parentHtmlUrl,omitempty"`
- CreatedDate time.Time `json:"createdDate" mapstructure:"-"`
+ CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
common.NoPKModel `json:"-" mapstructure:"-"`
}
diff --git a/plugins/github/tasks/account_collector.go
b/plugins/github/tasks/account_collector.go
index 79cabb118..49184e0ef 100644
--- a/plugins/github/tasks/account_collector.go
+++ b/plugins/github/tasks/account_collector.go
@@ -45,7 +45,7 @@ func CollectAccounts(taskCtx core.SubTaskContext)
errors.Error {
cursor, err := db.Cursor(
dal.Select("login"),
dal.From(models.GithubRepoAccount{}.TableName()),
- dal.Where("repo_github_id = ? and connection_id=?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_github_id = ? and connection_id=?",
data.Options.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
diff --git a/plugins/github/tasks/account_convertor.go
b/plugins/github/tasks/account_convertor.go
index 328de4a05..bd972d452 100644
--- a/plugins/github/tasks/account_convertor.go
+++ b/plugins/github/tasks/account_convertor.go
@@ -57,7 +57,7 @@ func ConvertAccounts(taskCtx core.SubTaskContext)
errors.Error {
dal.From(&githubModels.GithubAccount{}),
dal.Where(
"repo_github_id = ? and
_tool_github_accounts.connection_id=?",
- data.Repo.GithubId,
+ data.Options.GithubId,
data.Options.ConnectionId,
),
dal.Join(`left join _tool_github_repo_accounts gra on (
diff --git a/plugins/github/tasks/account_org_collector.go
b/plugins/github/tasks/account_org_collector.go
index fc148f439..76a62366d 100644
--- a/plugins/github/tasks/account_org_collector.go
+++ b/plugins/github/tasks/account_org_collector.go
@@ -52,7 +52,7 @@ func CollectAccountOrg(taskCtx core.SubTaskContext)
errors.Error {
AND ga.type = 'User'
)`),
dal.Where("_tool_github_repo_accounts.repo_github_id = ? and
_tool_github_repo_accounts.connection_id=?",
- data.Repo.GithubId, data.Options.ConnectionId),
+ data.Options.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
diff --git a/plugins/github/tasks/cicd_job_collector.go
b/plugins/github/tasks/cicd_job_collector.go
index c95c09525..2c748a1df 100644
--- a/plugins/github/tasks/cicd_job_collector.go
+++ b/plugins/github/tasks/cicd_job_collector.go
@@ -48,7 +48,7 @@ func CollectJobs(taskCtx core.SubTaskContext) errors.Error {
cursor, err := db.Cursor(
dal.Select("id"),
dal.From(models.GithubRun{}.TableName()),
- dal.Where("repo_id = ? and connection_id = ?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_id = ? and connection_id = ?",
data.Options.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
diff --git a/plugins/github/tasks/cicd_job_convertor.go
b/plugins/github/tasks/cicd_job_convertor.go
index 3e69e0100..774c2ac05 100644
--- a/plugins/github/tasks/cicd_job_convertor.go
+++ b/plugins/github/tasks/cicd_job_convertor.go
@@ -49,7 +49,7 @@ type SimpleBranch struct {
func ConvertJobs(taskCtx core.SubTaskContext) (err errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
deploymentPattern := data.Options.DeploymentPattern
productionPattern := data.Options.ProductionPattern
regexEnricher := helper.NewRegexEnricher()
diff --git a/plugins/github/tasks/cicd_job_extractor.go
b/plugins/github/tasks/cicd_job_extractor.go
index 8ac8c0926..0a2ab0397 100644
--- a/plugins/github/tasks/cicd_job_extractor.go
+++ b/plugins/github/tasks/cicd_job_extractor.go
@@ -35,7 +35,7 @@ var ExtractJobsMeta = core.SubTaskMeta{
func ExtractJobs(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
diff --git a/plugins/github/tasks/cicd_run_convertor.go
b/plugins/github/tasks/cicd_run_convertor.go
index 23a4b4e53..03a51af4c 100644
--- a/plugins/github/tasks/cicd_run_convertor.go
+++ b/plugins/github/tasks/cicd_run_convertor.go
@@ -42,7 +42,7 @@ var ConvertRunsMeta = core.SubTaskMeta{
func ConvertRuns(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
pipeline := &models.GithubRun{}
cursor, err := db.Cursor(
diff --git a/plugins/github/tasks/cicd_run_extractor.go
b/plugins/github/tasks/cicd_run_extractor.go
index a6d63d224..7b75d7ce1 100644
--- a/plugins/github/tasks/cicd_run_extractor.go
+++ b/plugins/github/tasks/cicd_run_extractor.go
@@ -35,7 +35,7 @@ var ExtractRunsMeta = core.SubTaskMeta{
func ExtractRuns(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
diff --git a/plugins/github/tasks/comment_extractor.go
b/plugins/github/tasks/comment_extractor.go
index b237d53ad..5c6553abe 100644
--- a/plugins/github/tasks/comment_extractor.go
+++ b/plugins/github/tasks/comment_extractor.go
@@ -79,14 +79,14 @@ func ExtractApiComments(taskCtx core.SubTaskContext)
errors.Error {
return nil, err
}
issue := &models.GithubIssue{}
- err = taskCtx.GetDal().All(issue,
dal.Where("connection_id = ? and number = ? and repo_id = ?",
data.Options.ConnectionId, issueINumber, data.Repo.GithubId))
+ err = taskCtx.GetDal().All(issue,
dal.Where("connection_id = ? and number = ? and repo_id = ?",
data.Options.ConnectionId, issueINumber, data.Options.GithubId))
if err != nil && !goerror.Is(err,
gorm.ErrRecordNotFound) {
return nil, err
}
//if we can not find issues with issue number above,
move the comments to github_pull_request_comments
if issue.GithubId == 0 {
pr := &models.GithubPullRequest{}
- err = taskCtx.GetDal().First(pr,
dal.Where("connection_id = ? and number = ? and repo_id = ?",
data.Options.ConnectionId, issueINumber, data.Repo.GithubId))
+ err = taskCtx.GetDal().First(pr,
dal.Where("connection_id = ? and number = ? and repo_id = ?",
data.Options.ConnectionId, issueINumber, data.Options.GithubId))
if err != nil && !goerror.Is(err,
gorm.ErrRecordNotFound) {
return nil, err
}
@@ -120,7 +120,7 @@ func ExtractApiComments(taskCtx core.SubTaskContext)
errors.Error {
results = append(results, githubIssueComment)
}
if apiComment.User != nil {
- githubAccount, err :=
convertAccount(apiComment.User, data.Repo.GithubId, data.Options.ConnectionId)
+ githubAccount, err :=
convertAccount(apiComment.User, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github/tasks/commit_convertor.go
b/plugins/github/tasks/commit_convertor.go
index f7229faa2..d52369485 100644
--- a/plugins/github/tasks/commit_convertor.go
+++ b/plugins/github/tasks/commit_convertor.go
@@ -41,7 +41,7 @@ var ConvertCommitsMeta = core.SubTaskMeta{
func ConvertCommits(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From("_tool_github_commits gc"),
diff --git a/plugins/github/tasks/commit_extractor.go
b/plugins/github/tasks/commit_extractor.go
index ae1e38cf1..150c8e8f6 100644
--- a/plugins/github/tasks/commit_extractor.go
+++ b/plugins/github/tasks/commit_extractor.go
@@ -109,7 +109,7 @@ func ExtractApiCommits(taskCtx core.SubTaskContext)
errors.Error {
githubRepoCommit := &models.GithubRepoCommit{
ConnectionId: data.Options.ConnectionId,
- RepoId: data.Repo.GithubId,
+ RepoId: data.Options.GithubId,
CommitSha: commit.Sha,
}
diff --git a/plugins/github/tasks/commit_stats_collector.go
b/plugins/github/tasks/commit_stats_collector.go
index e668443d3..97ed5825d 100644
--- a/plugins/github/tasks/commit_stats_collector.go
+++ b/plugins/github/tasks/commit_stats_collector.go
@@ -52,7 +52,7 @@ func CollectApiCommitStats(taskCtx core.SubTaskContext)
errors.Error {
err := db.First(
&latestUpdated,
dal.Join("left join _tool_github_repo_commits on
_tool_github_commit_stats.sha = _tool_github_repo_commits.commit_sha"),
- dal.Where("_tool_github_repo_commits.repo_id = ? and
_tool_github_repo_commits.connection_id = ?", data.Repo.GithubId,
data.Repo.ConnectionId),
+ dal.Where("_tool_github_repo_commits.repo_id = ? and
_tool_github_repo_commits.connection_id = ?", data.Options.GithubId,
data.Options.ConnectionId),
dal.Orderby("committed_date DESC"),
dal.Limit(1),
)
@@ -64,7 +64,7 @@ func CollectApiCommitStats(taskCtx core.SubTaskContext)
errors.Error {
dal.Join("left join _tool_github_repo_commits on
_tool_github_commits.sha = _tool_github_repo_commits.commit_sha"),
dal.From(models.GithubCommit{}.TableName()),
dal.Where("_tool_github_repo_commits.repo_id = ? and
_tool_github_repo_commits.connection_id = ? and
_tool_github_commits.committed_date >= ?",
- data.Repo.GithubId, data.Repo.ConnectionId,
latestUpdated.CommittedDate.String()),
+ data.Options.GithubId, data.Options.ConnectionId,
latestUpdated.CommittedDate.String()),
)
if err != nil {
return err
diff --git a/plugins/github/tasks/event_extractor.go
b/plugins/github/tasks/event_extractor.go
index 1cb6a1064..c661e0203 100644
--- a/plugins/github/tasks/event_extractor.go
+++ b/plugins/github/tasks/event_extractor.go
@@ -77,7 +77,7 @@ func ExtractApiEvents(taskCtx core.SubTaskContext)
errors.Error {
if body.Actor != nil {
githubIssueEvent.AuthorUsername =
body.Actor.Login
- githubAccount, err :=
convertAccount(body.Actor, data.Repo.GithubId, data.Options.ConnectionId)
+ githubAccount, err :=
convertAccount(body.Actor, data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github/tasks/issue_comment_convertor.go
b/plugins/github/tasks/issue_comment_convertor.go
index 50c9b15b7..7012f3cdc 100644
--- a/plugins/github/tasks/issue_comment_convertor.go
+++ b/plugins/github/tasks/issue_comment_convertor.go
@@ -41,7 +41,7 @@ var ConvertIssueCommentsMeta = core.SubTaskMeta{
func ConvertIssueComments(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubIssueComment{}),
diff --git a/plugins/github/tasks/issue_convertor.go
b/plugins/github/tasks/issue_convertor.go
index 154580489..feb9a707f 100644
--- a/plugins/github/tasks/issue_convertor.go
+++ b/plugins/github/tasks/issue_convertor.go
@@ -44,7 +44,7 @@ var ConvertIssuesMeta = core.SubTaskMeta{
func ConvertIssues(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
issue := &githubModels.GithubIssue{}
cursor, err := db.Cursor(
diff --git a/plugins/github/tasks/issue_extractor.go
b/plugins/github/tasks/issue_extractor.go
index 38ff4f569..8ae8d03b1 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -109,7 +109,7 @@ func ExtractApiIssues(taskCtx core.SubTaskContext)
errors.Error {
}
results := make([]interface{}, 0, 2)
- githubIssue, err := convertGithubIssue(body,
data.Options.ConnectionId, data.Repo.GithubId)
+ githubIssue, err := convertGithubIssue(body,
data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
return nil, err
}
@@ -122,7 +122,7 @@ func ExtractApiIssues(taskCtx core.SubTaskContext)
errors.Error {
if body.Assignee != nil {
githubIssue.AssigneeId = body.Assignee.Id
githubIssue.AssigneeName = body.Assignee.Login
- relatedUser, err :=
convertAccount(body.Assignee, data.Repo.GithubId, data.Options.ConnectionId)
+ relatedUser, err :=
convertAccount(body.Assignee, data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
@@ -131,7 +131,7 @@ func ExtractApiIssues(taskCtx core.SubTaskContext)
errors.Error {
if body.User != nil {
githubIssue.AuthorId = body.User.Id
githubIssue.AuthorName = body.User.Login
- relatedUser, err := convertAccount(body.User,
data.Repo.GithubId, data.Options.ConnectionId)
+ relatedUser, err := convertAccount(body.User,
data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github/tasks/issue_label_convertor.go
b/plugins/github/tasks/issue_label_convertor.go
index f0c9afdda..7abc9facc 100644
--- a/plugins/github/tasks/issue_label_convertor.go
+++ b/plugins/github/tasks/issue_label_convertor.go
@@ -41,7 +41,7 @@ var ConvertIssueLabelsMeta = core.SubTaskMeta{
func ConvertIssueLabels(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubIssueLabel{}),
diff --git a/plugins/github/tasks/milestone_converter.go
b/plugins/github/tasks/milestone_converter.go
index 4625b86df..e86523ff9 100644
--- a/plugins/github/tasks/milestone_converter.go
+++ b/plugins/github/tasks/milestone_converter.go
@@ -46,7 +46,7 @@ type MilestoneConverterModel struct {
func ConvertMilestones(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
connectionId := data.Options.ConnectionId
db := taskCtx.GetDal()
clauses := []dal.Clause{
diff --git a/plugins/github/tasks/milestone_extractor.go
b/plugins/github/tasks/milestone_extractor.go
index b473662c4..a8d4a4c48 100644
--- a/plugins/github/tasks/milestone_extractor.go
+++ b/plugins/github/tasks/milestone_extractor.go
@@ -90,7 +90,7 @@ func ExtractMilestones(taskCtx core.SubTaskContext)
errors.Error {
return nil, err
}
results := make([]interface{}, 0, 1)
- results = append(results,
convertGithubMilestone(response, data.Options.ConnectionId, data.Repo.GithubId))
+ results = append(results,
convertGithubMilestone(response, data.Options.ConnectionId,
data.Options.GithubId))
return results, nil
},
})
diff --git a/plugins/github/tasks/pr_comment_convertor.go
b/plugins/github/tasks/pr_comment_convertor.go
index a6b3e6a64..a3a5da6a0 100644
--- a/plugins/github/tasks/pr_comment_convertor.go
+++ b/plugins/github/tasks/pr_comment_convertor.go
@@ -41,7 +41,7 @@ var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
func ConvertPullRequestComments(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubPrComment{}),
diff --git a/plugins/github/tasks/pr_commit_collector.go
b/plugins/github/tasks/pr_commit_collector.go
index dcd5e02fd..f369377b7 100644
--- a/plugins/github/tasks/pr_commit_collector.go
+++ b/plugins/github/tasks/pr_commit_collector.go
@@ -72,7 +72,7 @@ func CollectApiPullRequestCommits(taskCtx
core.SubTaskContext) errors.Error {
clauses := []dal.Clause{
dal.Select("number, github_id"),
dal.From(models.GithubPullRequest{}.TableName()),
- dal.Where("repo_id = ? and connection_id=?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_id = ? and connection_id=?",
data.Options.GithubId, data.Options.ConnectionId),
}
if collectorWithState.CreatedDateAfter != nil {
clauses = append(clauses, dal.Where("github_created_at > ?",
*collectorWithState.CreatedDateAfter))
diff --git a/plugins/github/tasks/pr_commit_convertor.go
b/plugins/github/tasks/pr_commit_convertor.go
index a91cb0731..99c38c779 100644
--- a/plugins/github/tasks/pr_commit_convertor.go
+++ b/plugins/github/tasks/pr_commit_convertor.go
@@ -41,7 +41,7 @@ var ConvertPullRequestCommitsMeta = core.SubTaskMeta{
func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err errors.Error)
{
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
pullIdGen :=
didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
diff --git a/plugins/github/tasks/pr_commit_extractor.go
b/plugins/github/tasks/pr_commit_extractor.go
index f4af23978..41685d4f8 100644
--- a/plugins/github/tasks/pr_commit_extractor.go
+++ b/plugins/github/tasks/pr_commit_extractor.go
@@ -58,7 +58,7 @@ type PullRequestCommit struct {
func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
diff --git a/plugins/github/tasks/pr_convertor.go
b/plugins/github/tasks/pr_convertor.go
index 7d3b92d3b..e5573b31a 100644
--- a/plugins/github/tasks/pr_convertor.go
+++ b/plugins/github/tasks/pr_convertor.go
@@ -42,7 +42,7 @@ var ConvertPullRequestsMeta = core.SubTaskMeta{
func ConvertPullRequests(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&models.GithubPullRequest{}),
diff --git a/plugins/github/tasks/pr_extractor.go
b/plugins/github/tasks/pr_extractor.go
index 298115bb2..0d1ebeb8f 100644
--- a/plugins/github/tasks/pr_extractor.go
+++ b/plugins/github/tasks/pr_extractor.go
@@ -113,12 +113,12 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext)
errors.Error {
return nil, nil
}
//If this is a pr, ignore
- githubPr, err := convertGithubPullRequest(rawL,
data.Options.ConnectionId, data.Repo.GithubId)
+ githubPr, err := convertGithubPullRequest(rawL,
data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
return nil, err
}
if rawL.User != nil {
- githubUser, err := convertAccount(rawL.User,
data.Repo.GithubId, data.Options.ConnectionId)
+ githubUser, err := convertAccount(rawL.User,
data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github/tasks/pr_issue_convertor.go
b/plugins/github/tasks/pr_issue_convertor.go
index bbee9f71e..555433fb4 100644
--- a/plugins/github/tasks/pr_issue_convertor.go
+++ b/plugins/github/tasks/pr_issue_convertor.go
@@ -41,7 +41,7 @@ var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
func ConvertPullRequestIssues(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubPrIssue{}),
diff --git a/plugins/github/tasks/pr_issue_enricher.go
b/plugins/github/tasks/pr_issue_enricher.go
index ca5855db3..4eb1bf3fb 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -41,7 +41,7 @@ var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
var prBodyCloseRegex *regexp.Regexp
prBodyClosePattern := data.Options.PrBodyClosePattern
diff --git a/plugins/github/tasks/pr_label_convertor.go
b/plugins/github/tasks/pr_label_convertor.go
index 605080a60..8734a4d59 100644
--- a/plugins/github/tasks/pr_label_convertor.go
+++ b/plugins/github/tasks/pr_label_convertor.go
@@ -41,7 +41,7 @@ var ConvertPullRequestLabelsMeta = core.SubTaskMeta{
func ConvertPullRequestLabels(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubPrLabel{}),
diff --git a/plugins/github/tasks/pr_review_collector.go
b/plugins/github/tasks/pr_review_collector.go
index e793ac2c3..c003f2ce8 100644
--- a/plugins/github/tasks/pr_review_collector.go
+++ b/plugins/github/tasks/pr_review_collector.go
@@ -64,7 +64,7 @@ func CollectApiPullRequestReviews(taskCtx
core.SubTaskContext) errors.Error {
clauses := []dal.Clause{
dal.Select("number, github_id"),
dal.From(models.GithubPullRequest{}.TableName()),
- dal.Where("repo_id = ? and connection_id=?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_id = ? and connection_id=?",
data.Options.GithubId, data.Options.ConnectionId),
}
if collectorWithState.CreatedDateAfter != nil {
clauses = append(clauses, dal.Where("github_created_at > ?",
*collectorWithState.CreatedDateAfter))
diff --git a/plugins/github/tasks/pr_review_comment_extractor.go
b/plugins/github/tasks/pr_review_comment_extractor.go
index 216f4a7f3..b3a213307 100644
--- a/plugins/github/tasks/pr_review_comment_extractor.go
+++ b/plugins/github/tasks/pr_review_comment_extractor.go
@@ -100,7 +100,7 @@ func ExtractApiPrReviewComments(taskCtx
core.SubTaskContext) errors.Error {
githubPrComment.AuthorUserId =
prReviewComment.User.Id
githubPrComment.AuthorUsername =
prReviewComment.User.Login
- githubAccount, err :=
convertAccount(prReviewComment.User, data.Repo.GithubId,
data.Options.ConnectionId)
+ githubAccount, err :=
convertAccount(prReviewComment.User, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
@@ -127,7 +127,7 @@ func enrichGithubPrComment(data *GithubTaskData, db
dal.Dal, prUrlRegex *regexp.
return 0, errors.Default.Wrap(err, "parse prId failed")
}
pr := &models.GithubPullRequest{}
- err = db.First(pr, dal.Where("connection_id = ? and number = ?
and repo_id = ?", data.Options.ConnectionId, prNumber, data.Repo.GithubId))
+ err = db.First(pr, dal.Where("connection_id = ? and number = ?
and repo_id = ?", data.Options.ConnectionId, prNumber, data.Options.GithubId))
if goerror.Is(err, gorm.ErrRecordNotFound) {
return 0, nil
} else if err != nil {
diff --git a/plugins/github/tasks/pr_review_convertor.go
b/plugins/github/tasks/pr_review_convertor.go
index f427ae981..79a18099e 100644
--- a/plugins/github/tasks/pr_review_convertor.go
+++ b/plugins/github/tasks/pr_review_convertor.go
@@ -41,7 +41,7 @@ var ConvertPullRequestReviewsMeta = core.SubTaskMeta{
func ConvertPullRequestReviews(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&githubModels.GithubPrReview{}),
diff --git a/plugins/github/tasks/pr_review_extractor.go
b/plugins/github/tasks/pr_review_extractor.go
index f199c4a43..e72880e32 100644
--- a/plugins/github/tasks/pr_review_extractor.go
+++ b/plugins/github/tasks/pr_review_extractor.go
@@ -105,7 +105,7 @@ func ExtractApiPullRequestReviews(taskCtx
core.SubTaskContext) errors.Error {
githubPrReview.AuthorUserId =
apiPullRequestReview.User.Id
githubPrReview.AuthorUsername =
apiPullRequestReview.User.Login
- githubUser, err :=
convertAccount(apiPullRequestReview.User, data.Repo.GithubId,
data.Options.ConnectionId)
+ githubUser, err :=
convertAccount(apiPullRequestReview.User, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github/tasks/repo_convertor.go
b/plugins/github/tasks/repo_convertor.go
index f7d569853..b5493d56b 100644
--- a/plugins/github/tasks/repo_convertor.go
+++ b/plugins/github/tasks/repo_convertor.go
@@ -41,6 +41,7 @@ type ApiRepoResponse GithubApiRepo
type GithubApiRepo struct {
Name string `json:"name"`
+ FullName string `json:"full_name"`
GithubId int `json:"id"`
HTMLUrl string `json:"html_url"`
Language string `json:"language"`
@@ -63,7 +64,7 @@ var ConvertRepoMeta = core.SubTaskMeta{
func ConvertRepo(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
+ repoId := data.Options.GithubId
cursor, err := db.Cursor(
dal.From(&models.GithubRepo{}),
@@ -99,10 +100,14 @@ func ConvertRepo(taskCtx core.SubTaskContext) errors.Error
{
Description: repository.Description,
ForkedFrom: repository.ParentHTMLUrl,
Language: repository.Language,
- CreatedDate: &repository.CreatedDate,
+ CreatedDate: repository.CreatedDate,
UpdatedDate: repository.UpdatedDate,
}
-
+ if repository.OwnerLogin != "" {
+ domainRepository.Name = fmt.Sprintf("%s/%s",
repository.OwnerLogin, repository.Name)
+ } else {
+ domainRepository.Name = repository.Name
+ }
domainBoard := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
Id:
repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId),
@@ -110,7 +115,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) errors.Error {
Name: fmt.Sprintf("%s/%s",
repository.OwnerLogin, repository.Name),
Url: fmt.Sprintf("%s/%s",
repository.HTMLUrl, "issues"),
Description: repository.Description,
- CreatedDate: &repository.CreatedDate,
+ CreatedDate: repository.CreatedDate,
}
domainBoardRepo := &crossdomain.BoardRepo{
@@ -125,7 +130,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) errors.Error {
Name: repository.Name,
Url: repository.HTMLUrl,
Description: repository.Description,
- CreatedDate: &repository.CreatedDate,
+ CreatedDate: repository.CreatedDate,
UpdatedDate: repository.UpdatedDate,
}
diff --git a/plugins/github/tasks/task_data.go
b/plugins/github/tasks/task_data.go
index 25e4832f0..9fcedbf5c 100644
--- a/plugins/github/tasks/task_data.go
+++ b/plugins/github/tasks/task_data.go
@@ -30,6 +30,7 @@ type GithubOptions struct {
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId,omitempty"`
TransformationRuleId uint64 `json:"transformationRuleId"
mapstructure:"transformationRuleId,omitempty"`
ScopeId string `json:"scopeId"
mapstructure:"scopeId,omitempty"`
+ GithubId int `json:"githubId"
mapstructure:"githubId,omitempty"`
Tasks []string `json:"tasks,omitempty"
mapstructure:",omitempty"`
CreatedDateAfter string `json:"createdDateAfter"
mapstructure:"createdDateAfter,omitempty"`
Owner string `json:"owner"
mapstructure:"owner,omitempty"`
@@ -42,7 +43,6 @@ type GithubTaskData struct {
ApiClient *helper.ApiAsyncClient
GraphqlClient *helper.GraphqlAsyncClient
CreatedDateAfter *time.Time
- Repo *models.GithubRepo
}
type GithubApiParams struct {
diff --git a/plugins/github_graphql/impl/impl.go
b/plugins/github_graphql/impl/impl.go
index e58a8a6bd..2d40445a1 100644
--- a/plugins/github_graphql/impl/impl.go
+++ b/plugins/github_graphql/impl/impl.go
@@ -19,15 +19,14 @@ package impl
import (
"context"
- goerror "errors"
"fmt"
+ githubImpl "github.com/apache/incubator-devlake/plugins/github/impl"
"reflect"
"strings"
"time"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/apache/incubator-devlake/plugins/github_graphql/tasks"
@@ -147,7 +146,12 @@ func (plugin GithubGraphql) PrepareTaskData(taskCtx
core.TaskContext, options ma
return nil, errors.Default.Wrap(err, "unable to get github
connection by the given connection ID: %v")
}
- githubRepo, err := EnrichOptions(taskCtx, &op, connection)
+ apiClient, err := githubTasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get github API
client instance")
+ }
+
+ err = githubImpl.EnrichOptions(taskCtx, &op, apiClient.ApiClient)
if err != nil {
return nil, err
}
@@ -188,16 +192,10 @@ func (plugin GithubGraphql) PrepareTaskData(taskCtx
core.TaskContext, options ma
return
int(v.Elem().FieldByName(`RateLimit`).FieldByName(`Cost`).Int())
})
- apiClient, err := githubTasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get github API
client instance")
- }
-
taskData := &githubTasks.GithubTaskData{
Options: &op,
ApiClient: apiClient,
GraphqlClient: graphqlClient,
- Repo: githubRepo,
}
if !createdDateAfter.IsZero() {
taskData.CreatedDateAfter = &createdDateAfter
@@ -207,49 +205,6 @@ func (plugin GithubGraphql) PrepareTaskData(taskCtx
core.TaskContext, options ma
return taskData, nil
}
-func EnrichOptions(taskCtx core.TaskContext,
- op *githubTasks.GithubOptions,
- connection *models.GithubConnection) (*models.GithubRepo, errors.Error)
{
- var githubRepo models.GithubRepo
- var err errors.Error
- log := taskCtx.GetLogger()
- // for advanced mode or others which we already set value to onwer/repo
- if op.Owner != "" && op.Repo != "" {
- err := taskCtx.GetDal().First(&githubRepo, dal.Where(
- "connection_id = ? AND name = ? AND owner_login = ?",
- op.ConnectionId, op.Repo, op.Owner))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo %s/%s", op.Owner, op.Repo))
- }
- op.TransformationRuleId = githubRepo.TransformationRuleId
- }
- // for bp v200 which we only set ScopeId for options
- if githubRepo.GithubId == 0 && op.ScopeId != "" {
- log.Debug(fmt.Sprintf("Getting githubRepo by op.ScopeId: %s",
op.ScopeId))
- err = taskCtx.GetDal().First(&githubRepo,
dal.Where(`connection_id = ? AND github_id = ?`, connection.ID, op.ScopeId))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo %s", op.ScopeId))
- }
- ownerName := strings.Split(githubRepo.Name, "/")
- if len(ownerName) != 2 {
- return nil, errors.Default.New("Fail to set owner/repo
for github options.")
- }
- op.Owner = ownerName[0]
- op.Repo = ownerName[1]
- op.TransformationRuleId = githubRepo.TransformationRuleId
- }
- // Set GithubTransformationRule if it's nil, this has lower priority
- if op.GithubTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.GithubTransformationRule
- err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", githubRepo.TransformationRuleId))
- if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
- return nil, errors.BadInput.Wrap(err, "fail to get
transformationRule")
- }
- op.GithubTransformationRule = &transformationRule
- }
- return &githubRepo, nil
-}
-
// PkgPath information lost when compiled as plugin(.so)
func (plugin GithubGraphql) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/githubGraphql"
diff --git a/plugins/github_graphql/tasks/account_collector.go
b/plugins/github_graphql/tasks/account_collector.go
index 4b6c0d460..6bafccc7a 100644
--- a/plugins/github_graphql/tasks/account_collector.go
+++ b/plugins/github_graphql/tasks/account_collector.go
@@ -77,7 +77,7 @@ func CollectAccount(taskCtx core.SubTaskContext) errors.Error
{
cursor, err := db.Cursor(
dal.Select("login"),
dal.From(models.GithubRepoAccount{}.TableName()),
- dal.Where("repo_github_id = ? and connection_id=?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_github_id = ? and connection_id=?",
data.Options.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
diff --git a/plugins/github_graphql/tasks/check_run_collector.go
b/plugins/github_graphql/tasks/check_run_collector.go
index 9dabaffb2..fe25a63ed 100644
--- a/plugins/github_graphql/tasks/check_run_collector.go
+++ b/plugins/github_graphql/tasks/check_run_collector.go
@@ -102,7 +102,7 @@ func CollectCheckRun(taskCtx core.SubTaskContext)
errors.Error {
cursor, err := db.Cursor(
dal.Select("check_suite_node_id"),
dal.From(models.GithubRun{}.TableName()),
- dal.Where("repo_id = ? and connection_id=?",
data.Repo.GithubId, data.Options.ConnectionId),
+ dal.Where("repo_id = ? and connection_id=?",
data.Options.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
@@ -159,7 +159,7 @@ func CollectCheckRun(taskCtx core.SubTaskContext)
errors.Error {
githubJob := &models.GithubJob{
ConnectionId:
data.Options.ConnectionId,
RunID:
node.CheckSuite.WorkflowRun.DatabaseId,
- RepoId:
data.Repo.GithubId,
+ RepoId:
data.Options.GithubId,
ID:
checkRun.DatabaseId,
NodeID: checkRun.Id,
HTMLURL:
checkRun.DetailsUrl,
diff --git a/plugins/github_graphql/tasks/issue_collector.go
b/plugins/github_graphql/tasks/issue_collector.go
index 8bd8f6a5f..402515153 100644
--- a/plugins/github_graphql/tasks/issue_collector.go
+++ b/plugins/github_graphql/tasks/issue_collector.go
@@ -91,7 +91,7 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
return nil
}
- milestoneMap, err := getMilestoneMap(db, data.Repo.GithubId,
data.Repo.ConnectionId)
+ milestoneMap, err := getMilestoneMap(db, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil
}
@@ -133,7 +133,7 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error
{
isFinish = true
break
}
- githubIssue, err :=
convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId,
data.Repo.GithubId)
+ githubIssue, err :=
convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId,
data.Options.GithubId)
if err != nil {
return nil, err
}
@@ -144,14 +144,14 @@ func CollectIssue(taskCtx core.SubTaskContext)
errors.Error {
results = append(results, githubLabels...)
results = append(results, githubIssue)
if issue.AssigneeList.Assignees != nil &&
len(issue.AssigneeList.Assignees) > 0 {
- relatedUser, err :=
convertGraphqlPreAccount(issue.AssigneeList.Assignees[0], data.Repo.GithubId,
data.Options.ConnectionId)
+ relatedUser, err :=
convertGraphqlPreAccount(issue.AssigneeList.Assignees[0],
data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
results = append(results, relatedUser)
}
if issue.Author != nil {
- relatedUser, err :=
convertGraphqlPreAccount(*issue.Author, data.Repo.GithubId,
data.Options.ConnectionId)
+ relatedUser, err :=
convertGraphqlPreAccount(*issue.Author, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github_graphql/tasks/pr_collector.go
b/plugins/github_graphql/tasks/pr_collector.go
index 53f1045e7..1a3a69689 100644
--- a/plugins/github_graphql/tasks/pr_collector.go
+++ b/plugins/github_graphql/tasks/pr_collector.go
@@ -183,12 +183,12 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
break
}
//If this is a pr, ignore
- githubPr, err := convertGithubPullRequest(rawL,
data.Options.ConnectionId, data.Repo.GithubId)
+ githubPr, err := convertGithubPullRequest(rawL,
data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
return nil, err
}
if rawL.Author != nil {
- githubUser, err :=
convertGraphqlPreAccount(*rawL.Author, data.Repo.GithubId,
data.Options.ConnectionId)
+ githubUser, err :=
convertGraphqlPreAccount(*rawL.Author, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
@@ -243,7 +243,7 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
githubPrReview.AuthorUserId = apiPullRequestReview.Author.Id
githubPrReview.AuthorUsername = apiPullRequestReview.Author.Login
- githubUser, err :=
convertGraphqlPreAccount(*apiPullRequestReview.Author, data.Repo.GithubId,
data.Options.ConnectionId)
+ githubUser, err :=
convertGraphqlPreAccount(*apiPullRequestReview.Author, data.Options.GithubId,
data.Options.ConnectionId)
if err != nil {
return nil, err
}
@@ -273,7 +273,7 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
results = append(results,
githubPullRequestCommit)
if
apiPullRequestCommit.Commit.Author.User != nil {
- githubUser, err :=
convertGraphqlPreAccount(*apiPullRequestCommit.Commit.Author.User,
data.Repo.GithubId, data.Options.ConnectionId)
+ githubUser, err :=
convertGraphqlPreAccount(*apiPullRequestCommit.Commit.Author.User,
data.Options.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
diff --git a/plugins/github_graphql/tasks/repo_collector.go
b/plugins/github_graphql/tasks/repo_collector.go
deleted file mode 100644
index 5547cd990..000000000
--- a/plugins/github_graphql/tasks/repo_collector.go
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/github/models"
- githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
- "github.com/apache/incubator-devlake/plugins/helper"
- "github.com/merico-dev/graphql"
- "time"
-)
-
-const RAW_REPO_TABLE = "github_graphql_repo"
-
-var _ core.SubTaskEntryPoint = CollectRepo
-
-type GraphqlQueryRepo struct {
- RateLimit struct {
- Cost int
- }
- Repository struct {
- Name string `graphql:"name"`
- GithubId int `graphql:"databaseId"`
- HTMLUrl string `graphql:"url"`
- Languages struct {
- Nodes []struct {
- Name string
- }
- } `graphql:"languages(first: 1)"`
- Description string `graphql:"description"`
- Owner GraphqlInlineAccountQuery
- CreatedDate time.Time `graphql:"createdAt"`
- UpdatedDate *time.Time `graphql:"updatedAt"`
- Parent *struct {
- GithubId int `graphql:"databaseId"`
- HTMLUrl string `graphql:"url"`
- }
- } `graphql:"repository(owner: $owner, name: $name)"`
-}
-
-var CollectRepoMeta = core.SubTaskMeta{
- Name: "CollectRepo",
- EntryPoint: CollectRepo,
- EnabledByDefault: false,
- Description: "Collect Repo data from GithubGraphql api",
- DomainTypes: []string{core.DOMAIN_TYPE_CODE,
core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CICD, core.DOMAIN_TYPE_CODE_REVIEW,
core.DOMAIN_TYPE_CROSS},
-}
-
-func CollectRepo(taskCtx core.SubTaskContext) errors.Error {
- data := taskCtx.GetData().(*githubTasks.GithubTaskData)
- db := taskCtx.GetDal()
-
- collector, err :=
helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
- set of data to be process, for example, we
process JiraIssues by Board
- */
- Params: githubTasks.GithubApiParams{
- ConnectionId: data.Options.ConnectionId,
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
- },
- Table: RAW_REPO_TABLE,
- },
- GraphqlClient: data.GraphqlClient,
- /*
- (Optional) Return query string for request, or you can
plug them into UrlTemplate directly
- */
- BuildQuery: func(reqData *helper.GraphqlRequestData)
(interface{}, map[string]interface{}, error) {
- query := &GraphqlQueryRepo{}
- variables := map[string]interface{}{
- "owner": graphql.String(data.Options.Owner),
- "name": graphql.String(data.Options.Repo),
- }
- return query, variables, nil
- },
- ResponseParser: func(iQuery interface{}, variables
map[string]interface{}) ([]interface{}, error) {
- query := iQuery.(*GraphqlQueryRepo)
- repository := query.Repository
- results := make([]interface{}, 0, 1)
- language := ``
- if len(repository.Languages.Nodes) > 0 {
- language = repository.Languages.Nodes[0].Name
- }
- githubRepository := &models.GithubRepo{
- ConnectionId: data.Options.ConnectionId,
- GithubId: repository.GithubId,
- Name: repository.Name,
- HTMLUrl: repository.HTMLUrl,
- Description: repository.Description,
- OwnerId: repository.Owner.Id,
- OwnerLogin: repository.Owner.Login,
- Language: language,
- CreatedDate: repository.CreatedDate,
- UpdatedDate: repository.UpdatedDate,
- }
- data.Repo = githubRepository
-
- if repository.Parent != nil {
- githubRepository.ParentGithubId =
repository.Parent.GithubId
- githubRepository.ParentHTMLUrl =
repository.Parent.HTMLUrl
- }
- err := db.CreateOrUpdate(githubRepository)
- if err != nil {
- return nil, err
- }
- results = append(results, githubRepository)
-
- githubUser, err :=
convertGraphqlPreAccount(repository.Owner, data.Repo.GithubId,
data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, githubUser)
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return collector.Execute()
-}
diff --git a/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 07fc19f0f..309699c10 100644
--- a/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+pipeline_id,commit_sha,branch,repo_id,repo,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
diff --git a/plugins/jenkins/e2e/builds_test.go
b/plugins/jenkins/e2e/builds_test.go
index a345213cf..33fcc91e8 100644
--- a/plugins/jenkins/e2e/builds_test.go
+++ b/plugins/jenkins/e2e/builds_test.go
@@ -143,7 +143,7 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
e2ehelper.ColumnWithRawData(
"pipeline_id",
"repo_id",
- "repo_url",
+ "repo",
"branch",
"commit_sha",
),
diff --git a/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 1027c0ab0..dec0c0ceb 100644
--- a/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,repo_id,repo_url,branch,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+pipeline_id,commit_sha,repo_id,repo,branch,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#11,ceeffdfdd06bce232f9adb3a656265bad13a8473,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,95,
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#21,0f886c74949c3ee7e489188911c7dc0c1d547418,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,94,
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#31,0f886c74949c3ee7e489188911c7dc0c1d547418,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,93,
diff --git a/plugins/jenkins/tasks/build_commit_convertor.go
b/plugins/jenkins/tasks/build_commit_convertor.go
index 906c27f3d..72ff17ad4 100644
--- a/plugins/jenkins/tasks/build_commit_convertor.go
+++ b/plugins/jenkins/tasks/build_commit_convertor.go
@@ -75,7 +75,7 @@ func ConvertBuildRepos(taskCtx core.SubTaskContext)
errors.Error {
PipelineId:
buildIdGen.Generate(jenkinsBuildCommit.ConnectionId,
jenkinsBuildCommit.BuildName),
CommitSha: jenkinsBuildCommit.CommitSha,
Branch: jenkinsBuildCommit.Branch,
- RepoUrl: jenkinsBuildCommit.RepoUrl,
+ Repo: jenkinsBuildCommit.RepoUrl,
}
return []interface{}{
build,
diff --git a/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
b/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
index 1567f87d2..b32b2ea3b 100644
--- a/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
+++ b/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url
+pipeline_id,commit_sha,branch,repo_id,repo
github:GithubRun:1:1,commit_sha1,main,github:GithubRepo:1:484251804,""
github:GithubRun:1:2,commit_sha2,main,github:GithubRepo:1:484251804,""
github:GithubRun:1:3,commit_sha3,fix-edit-url,github:GithubRepo:1:484251804,""