This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new dc941656 github repo task refactor (#2199)
dc941656 is described below
commit dc9416569b716367f722430f509d006dea3e7eb7
Author: likyh <[email protected]>
AuthorDate: Thu Jun 16 15:34:25 2022 +0800
github repo task refactor (#2199)
* fix repo
* append ConnectionId for repo in github
* add repo raw table
* add csv
* update csv for connectionId
* update csv for connectionId 2
Co-authored-by: linyh <[email protected]>
---
.../raw_tables/_raw_github_api_repositories.csv | 2 +
plugins/github/e2e/repo_test.go | 109 +++++++++++++++++++++
.../e2e/snapshot_tables/_tool_github_repos.csv | 2 +
plugins/github/e2e/snapshot_tables/boards.csv | 2 +
plugins/github/e2e/snapshot_tables/repos.csv | 2 +
.../models/migrationscripts/archived/repo.go | 1 +
plugins/github/models/repo.go | 1 +
plugins/github/tasks/issue_collector.go | 5 +-
plugins/github/tasks/repo_collector.go | 5 +-
plugins/github/tasks/repo_convertor.go | 19 ++--
plugins/github/tasks/repositorie_extractor.go | 30 +++---
11 files changed, 154 insertions(+), 24 deletions(-)
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_repositories.csv
b/plugins/github/e2e/raw_tables/_raw_github_api_repositories.csv
new file mode 100644
index 00000000..db7c8f34
--- /dev/null
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_repositories.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+7,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":134018330,""node_id"":""MDEwOlJlcG9zaXRvcnkxMzQwMTgzMzA="",""name"":""ants"",""full_name"":""panjf2000/ants"",""private"":false,""owner"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"",""followers_
[...]
diff --git a/plugins/github/e2e/repo_test.go b/plugins/github/e2e/repo_test.go
new file mode 100644
index 00000000..45612292
--- /dev/null
+++ b/plugins/github/e2e/repo_test.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
+
+func TestRepoDataFlow(t *testing.T) {
+ var plugin impl.Github
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", plugin)
+
+ githubRepository := &models.GithubRepo{
+ GithubId: 134018330,
+ }
+ taskData := &tasks.GithubTaskData{
+ Options: &tasks.GithubOptions{
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ Config: models.Config{
+ PrType: "type/(.*)$",
+ PrComponent: "component/(.*)$",
+ },
+ },
+ Repo: githubRepository,
+ }
+
+ // import raw data table
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_repositories.csv",
"_raw_github_api_repositories")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubRepo{})
+ dataflowTester.Subtask(tasks.ExtractApiRepoMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GithubRepo{},
+ fmt.Sprintf("./snapshot_tables/%s.csv",
models.GithubRepo{}.TableName()),
+ []string{"connection_id", "github_id"},
+ []string{
+ "name",
+ "html_url",
+ "description",
+ "owner_id",
+ "owner_login",
+ "language",
+ "created_date",
+ "updated_date",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify extraction
+ dataflowTester.FlushTabler(&code.Repo{})
+ dataflowTester.FlushTabler(&ticket.Board{})
+ dataflowTester.Subtask(tasks.ConvertRepoMeta, taskData)
+ dataflowTester.VerifyTable(
+ code.Repo{},
+ fmt.Sprintf("./snapshot_tables/%s.csv",
code.Repo{}.TableName()),
+ []string{"id"},
+ []string{
+ "name",
+ "url",
+ "description",
+ "owner_id",
+ "language",
+ "forked_from",
+ "created_date",
+ "updated_date",
+ "deleted",
+ },
+ )
+ dataflowTester.VerifyTable(
+ ticket.Board{},
+ fmt.Sprintf("./snapshot_tables/%s.csv",
ticket.Board{}.TableName()),
+ []string{"id"},
+ []string{
+ "name",
+ "description",
+ "url",
+ "created_date",
+ },
+ )
+}
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_repos.csv
b/plugins/github/e2e/snapshot_tables/_tool_github_repos.csv
new file mode 100644
index 00000000..20fc8dfa
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_repos.csv
@@ -0,0 +1,2 @@
+connection_id,github_id,name,html_url,description,owner_id,owner_login,language,created_date,updated_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,134018330,ants,https://github.com/panjf2000/ants,"πππ ants is a
high-performance and low-cost goroutine pool in Go, inspired by fasthttp./ ants
ζ―δΈδΈͺι«ζ§θ½δΈδ½ζθη goroutine
ζ± γ",7496278,panjf2000,Go,2018-05-19T01:13:38.000+00:00,2022-06-13T15:27:54.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_repositories,7,
diff --git a/plugins/github/e2e/snapshot_tables/boards.csv
b/plugins/github/e2e/snapshot_tables/boards.csv
new file mode 100644
index 00000000..600d1ce8
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/boards.csv
@@ -0,0 +1,2 @@
+id,name,description,url,created_date
+gitlab:GithubRepo:1:134018330,panjf2000/ants,"πππ ants is a high-performance
and low-cost goroutine pool in Go, inspired by fasthttp./ ants ζ―δΈδΈͺι«ζ§θ½δΈδ½ζθη
goroutine
ζ± γ",https://github.com/panjf2000/ants/issues,2018-05-19T01:13:38.000+00:00
diff --git a/plugins/github/e2e/snapshot_tables/repos.csv
b/plugins/github/e2e/snapshot_tables/repos.csv
new file mode 100644
index 00000000..5a6211a5
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/repos.csv
@@ -0,0 +1,2 @@
+id,name,url,description,owner_id,language,forked_from,created_date,updated_date,deleted
+gitlab:GithubRepo:1:134018330,panjf2000/ants,https://github.com/panjf2000/ants,"πππ
ants is a high-performance and low-cost goroutine pool in Go, inspired by
fasthttp./ ants ζ―δΈδΈͺι«ζ§θ½δΈδ½ζθη goroutine
ζ± γ",,Go,,2018-05-19T01:13:38.000+00:00,2022-06-13T15:27:54.000+00:00,0
diff --git a/plugins/github/models/migrationscripts/archived/repo.go
b/plugins/github/models/migrationscripts/archived/repo.go
index 7ecee73b..fd1433e5 100644
--- a/plugins/github/models/migrationscripts/archived/repo.go
+++ b/plugins/github/models/migrationscripts/archived/repo.go
@@ -24,6 +24,7 @@ import (
)
type GithubRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
HTMLUrl string `gorm:"type:varchar(255)"`
diff --git a/plugins/github/models/repo.go b/plugins/github/models/repo.go
index b52b443e..a60dbc36 100644
--- a/plugins/github/models/repo.go
+++ b/plugins/github/models/repo.go
@@ -23,6 +23,7 @@ import (
)
type GithubRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
HTMLUrl string `gorm:"type:varchar(255)"`
diff --git a/plugins/github/tasks/issue_collector.go
b/plugins/github/tasks/issue_collector.go
index 7535bf23..89f11cd0 100644
--- a/plugins/github/tasks/issue_collector.go
+++ b/plugins/github/tasks/issue_collector.go
@@ -33,8 +33,9 @@ const RAW_ISSUE_TABLE = "github_api_issues"
// this struct should be moved to `gitub_api_common.go`
type GithubApiParams struct {
- Owner string
- Repo string
+ ConnectionId uint64
+ Owner string
+ Repo string
}
var CollectApiIssuesMeta = core.SubTaskMeta{
diff --git a/plugins/github/tasks/repo_collector.go
b/plugins/github/tasks/repo_collector.go
index 5cb940f4..29544354 100644
--- a/plugins/github/tasks/repo_collector.go
+++ b/plugins/github/tasks/repo_collector.go
@@ -45,8 +45,9 @@ func CollectApiRepositories(taskCtx core.SubTaskContext)
error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_REPOSITORIES_TABLE,
},
diff --git a/plugins/github/tasks/repo_convertor.go
b/plugins/github/tasks/repo_convertor.go
index 96fbc513..4ad0d6bc 100644
--- a/plugins/github/tasks/repo_convertor.go
+++ b/plugins/github/tasks/repo_convertor.go
@@ -19,6 +19,7 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
"github.com/apache/incubator-devlake/models/domainlayer"
@@ -38,13 +39,14 @@ var ConvertRepoMeta = core.SubTaskMeta{
}
func ConvertRepo(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
- cursor, err := db.Model(&models.GithubRepo{}).
- Where("github_id = ?", repoId).
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GithubRepo{}),
+ dal.Where("github_id = ?", repoId),
+ )
if err != nil {
return err
}
@@ -58,8 +60,9 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_REPOSITORIES_TABLE,
},
@@ -67,7 +70,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
repository := inputRow.(*models.GithubRepo)
domainRepository := &code.Repo{
DomainEntity: domainlayer.DomainEntity{
- Id:
repoIdGen.Generate(repository.GithubId),
+ Id:
repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId),
},
Name: fmt.Sprintf("%s/%s",
repository.OwnerLogin, repository.Name),
Url: repository.HTMLUrl,
@@ -80,7 +83,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
domainBoard := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
- Id:
repoIdGen.Generate(repository.GithubId),
+ Id:
repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId),
},
Name: fmt.Sprintf("%s/%s",
repository.OwnerLogin, repository.Name),
Url: fmt.Sprintf("%s/%s",
repository.HTMLUrl, "issues"),
diff --git a/plugins/github/tasks/repositorie_extractor.go
b/plugins/github/tasks/repositorie_extractor.go
index 8a118299..27102cca 100644
--- a/plugins/github/tasks/repositorie_extractor.go
+++ b/plugins/github/tasks/repositorie_extractor.go
@@ -57,8 +57,9 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext)
error {
set of data to be process, for example, we
process JiraIssues by Board
*/
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
/*
Table store raw data
@@ -76,15 +77,16 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext)
error {
}
results := make([]interface{}, 0, 1)
githubRepository := &models.GithubRepo{
- GithubId: body.GithubId,
- Name: body.Name,
- HTMLUrl: body.HTMLUrl,
- Description: body.Description,
- OwnerId: body.Owner.Id,
- OwnerLogin: body.Owner.Login,
- Language: body.Language,
- CreatedDate: body.CreatedAt.ToTime(),
- UpdatedDate:
helper.Iso8601TimeToTime(body.UpdatedAt),
+ ConnectionId: data.Options.ConnectionId,
+ GithubId: body.GithubId,
+ Name: body.Name,
+ HTMLUrl: body.HTMLUrl,
+ Description: body.Description,
+ OwnerId: body.Owner.Id,
+ OwnerLogin: body.Owner.Login,
+ Language: body.Language,
+ CreatedDate: body.CreatedAt.ToTime(),
+ UpdatedDate:
helper.Iso8601TimeToTime(body.UpdatedAt),
}
data.Repo = githubRepository
@@ -93,7 +95,11 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext)
error {
githubRepository.ParentHTMLUrl =
body.Parent.HTMLUrl
}
results = append(results, githubRepository)
- taskCtx.TaskContext().GetData().(*GithubTaskData).Repo
= githubRepository
+
+ parentTaskContext := taskCtx.TaskContext()
+ if parentTaskContext != nil {
+
parentTaskContext.GetData().(*GithubTaskData).Repo = githubRepository
+ }
return results, nil
},
})
