This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f6957c258 Feat GitHub blueprint v200 (#3819)
f6957c258 is described below
commit f6957c25876a4eac0d7a21803c2c4e4b8b0c3da1
Author: Warren Chen <[email protected]>
AuthorDate: Wed Nov 30 14:46:42 2022 +0800
Feat GitHub blueprint v200 (#3819)
* feat(github): implement v200
* feat(github): implement blueprint v200
* feat(github): add comments
* feat(github): add more unit tests
---
plugins/core/plugin_blueprint.go | 5 +-
plugins/github/api/blueprint.go | 139 ++++++++------
.../{blueprint_test.go => blueprint_V200_test.go} | 99 ++++++----
plugins/github/api/blueprint_test.go | 110 +++++++----
plugins/github/api/blueprint_v200.go | 210 +++++++++++++++++++++
plugins/github/impl/impl.go | 4 +
plugins/github/models/repo.go | 28 +--
plugins/github/models/transformation_rule.go | 24 +--
8 files changed, 467 insertions(+), 152 deletions(-)
diff --git a/plugins/core/plugin_blueprint.go b/plugins/core/plugin_blueprint.go
index 3d8ca0583..1226a488a 100644
--- a/plugins/core/plugin_blueprint.go
+++ b/plugins/core/plugin_blueprint.go
@@ -133,8 +133,9 @@ type BlueprintConnectionV200 struct {
// BlueprintScopeV200 contains the `id` and `name` for a specific scope
// transformationRuleId should be deduced by the ScopeId
type BlueprintScopeV200 struct {
- Id string `json:"id"`
- Name string `json:"name"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Entities []string `json:"entities"`
}
// MetricPluginBlueprintV200 is similar to the DataSourcePluginBlueprintV200
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 41b9efa52..4dc771712 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -107,70 +107,24 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
if err != nil {
return nil, err
}
- memorizedGetApiRepo := func() (*tasks.GithubApiRepo,
errors.Error) {
- if repo == nil {
- repo, err = getApiRepo(op, apiClient)
- }
- return repo, err
- }
stage := plan[i]
if stage == nil {
stage = core.PipelineStage{}
}
- // construct github(graphql) task
- if connection.EnableGraphql {
- // FIXME this need fix when 2 plugins merged
- plugin, err := core.GetPlugin(`github_graphql`)
- if err != nil {
- return nil, err
- }
- if pluginGq, ok := plugin.(core.PluginTask); ok {
- subtasks, err :=
helper.MakePipelinePlanSubtasks(pluginGq.SubTaskMetas(), scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- stage = append(stage, &core.PipelineTask{
- Plugin: "github_graphql",
- Subtasks: subtasks,
- Options: options,
- })
- } else {
- return nil, errors.BadInput.New("plugin
github_graphql does not support SubTaskMetas")
- }
- } else {
- subtasks, err :=
helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- stage = append(stage, &core.PipelineTask{
- Plugin: "github",
- Subtasks: subtasks,
- Options: options,
- })
+ stage, err = addGithub(subtaskMetas, connection,
scopeElem.Entities, stage, options)
+ if err != nil {
+ return nil, err
}
// collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities,
core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo
id beforehand
- token := strings.Split(connection.Token, ",")[0]
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
- cloneUrl, err :=
errors.Convert01(url.Parse(repo.CloneUrl))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
repo.GithubId),
- "proxy": connection.Proxy,
- },
- })
+ repo, err = memorizedGetApiRepo(repo, op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ stage, err = addGitex(scopeElem.Entities, connection, repo,
stage)
+ if err != nil {
+ return nil, err
}
// dora
if productionPattern, ok :=
transformationRules["productionPattern"]; ok && productionPattern != nil {
@@ -185,7 +139,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
if j == len(plan) {
plan = append(plan, nil)
}
- repo, err = memorizedGetApiRepo()
+ repo, err = memorizedGetApiRepo(repo, op, apiClient)
if err != nil {
return nil, err
}
@@ -211,6 +165,66 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
return plan, nil
}
+func addGitex(entities []string,
+ connection *models.GithubConnection,
+ repo *tasks.GithubApiRepo,
+ stage core.PipelineStage,
+) (core.PipelineStage, errors.Error) {
+ if utils.StringsContains(entities, core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo id
beforehand
+ token := strings.Split(connection.Token, ",")[0]
+ cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl.User = url.UserPassword("git", token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
repo.GithubId),
+ "proxy": connection.Proxy,
+ },
+ })
+ }
+ return stage, nil
+}
+
+func addGithub(subtaskMetas []core.SubTaskMeta, connection
*models.GithubConnection, entities []string, stage core.PipelineStage, options
map[string]interface{}) (core.PipelineStage, errors.Error) {
+ // construct github(graphql) task
+ if connection.EnableGraphql {
+ // FIXME this need fix when 2 plugins merged
+ plugin, err := core.GetPlugin(`github_graphql`)
+ if err != nil {
+ return nil, err
+ }
+ if pluginGq, ok := plugin.(core.PluginTask); ok {
+ subtasks, err :=
helper.MakePipelinePlanSubtasks(pluginGq.SubTaskMetas(), entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "github_graphql",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ } else {
+ return nil, errors.BadInput.New("plugin github_graphql
does not support SubTaskMetas")
+ }
+ } else {
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ }
+ return stage, nil
+}
+
func getApiRepo(op *tasks.GithubOptions, apiClient helper.ApiClientGetter)
(*tasks.GithubApiRepo, errors.Error) {
apiRepo := &tasks.GithubApiRepo{}
res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner,
op.Repo), nil, nil)
@@ -231,3 +245,14 @@ func getApiRepo(op *tasks.GithubOptions, apiClient
helper.ApiClientGetter) (*tas
}
return apiRepo, nil
}
+
+func memorizedGetApiRepo(repo *tasks.GithubApiRepo, op *tasks.GithubOptions,
apiClient helper.ApiClientGetter) (*tasks.GithubApiRepo, errors.Error) {
+ if repo == nil {
+ var err errors.Error
+ repo, err = getApiRepo(op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return repo, nil
+}
diff --git a/plugins/github/api/blueprint_test.go
b/plugins/github/api/blueprint_V200_test.go
similarity index 62%
copy from plugins/github/api/blueprint_test.go
copy to plugins/github/api/blueprint_V200_test.go
index 9a35c14d9..ad0f67a3d 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_V200_test.go
@@ -22,6 +22,9 @@ import (
"encoding/json"
"github.com/apache/incubator-devlake/mocks"
"github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -31,9 +34,10 @@ import (
"io"
"net/http"
"testing"
+ "time"
)
-func TestProcessScope(t *testing.T) {
+func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
connection := &models.GithubConnection{
RestConnection: helper.RestConnection{
BaseConnection: helper.BaseConnection{
@@ -65,32 +69,48 @@ func TestProcessScope(t *testing.T) {
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
err = core.RegisterPlugin("github", mockMeta)
assert.Nil(t, err)
- bs := &core.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "owner": "test",
- "repo": "testRepo"
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
+ bs := &core.BlueprintScopeV200{
+ Entities: []string{"CODE", "TICKET"},
+ Id: "",
+ Name: "",
+ }
+ bpScopes := make([]*core.BlueprintScopeV200, 0)
+ bpScopes = append(bpScopes, bs)
+
+ plan := make(core.PipelinePlan, len(bpScopes))
+ scopes := make([]core.Scope, 0, len(bpScopes))
+ for i, bpScope := range bpScopes {
+ githubRepo := &models.GithubRepo{
+ ConnectionId: 1,
+ GithubId: 12345,
+ Name: "testRepo",
+ OwnerLogin: "test",
+ CreatedDate: time.Time{},
+ }
+
+ transformationRule := &models.TransformationRules{
+ Model: common.Model{
+ ID: 1,
+ },
+ PrType: "hey,man,wasup",
+ ReffdiffRule: json.RawMessage(`{
"tagsPattern": "pattern",
"tagsLimit": 10,
"tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
- scopes := make([]*core.BlueprintScopeV100, 0)
- scopes = append(scopes, bs)
- plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
- assert.Nil(t, err)
+ }`),
+ }
+ var scope []core.Scope
+ plan, scope, err = makeDataSourcePipelinePlanV200(nil, i, plan,
bpScope, connection, mockApiCLient, githubRepo, transformationRule)
+ assert.Nil(t, err)
+ scopes = append(scopes, scope...)
+ }
expectPlan := core.PipelinePlan{
core.PipelineStage{
{
- Plugin: "github",
- Subtasks: []string{},
+ Plugin: "github",
+ Subtasks: []string{},
+ SkipOnFail: false,
Options: map[string]interface{}{
"connectionId": uint64(1),
"owner": "test",
@@ -101,7 +121,8 @@ func TestProcessScope(t *testing.T) {
},
},
{
- Plugin: "gitextractor",
+ Plugin: "gitextractor",
+ SkipOnFail: false,
Options: map[string]interface{}{
"proxy": "",
"repoId": "github:GithubRepo:1:12345",
@@ -111,7 +132,8 @@ func TestProcessScope(t *testing.T) {
},
core.PipelineStage{
{
- Plugin: "refdiff",
+ Plugin: "refdiff",
+ SkipOnFail: false,
Options: map[string]interface{}{
"tagsLimit": float64(10),
"tagsOrder": "reverse semver",
@@ -119,18 +141,27 @@ func TestProcessScope(t *testing.T) {
},
},
},
- core.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{
- "repoId": "github:GithubRepo:1:12345",
- "transformationRules":
map[string]interface{}{
- "productionPattern": "xxxx",
- },
- },
- },
- },
}
assert.Equal(t, expectPlan, plan)
+ expectScopes := make([]core.Scope, 0)
+ scopeRepo := &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: "github:GithubRepo:1:12345",
+ },
+ Name: "test/testRepo",
+ }
+
+ scopeTicket := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: "github:GithubRepo:1:12345",
+ },
+ Name: "test/testRepo",
+ Description: "",
+ Url: "",
+ CreatedDate: nil,
+ Type: "",
+ }
+
+ expectScopes = append(expectScopes, scopeRepo, scopeTicket)
+ assert.Equal(t, expectScopes, scopes)
}
diff --git a/plugins/github/api/blueprint_test.go
b/plugins/github/api/blueprint_test.go
index 9a35c14d9..8bb4f3df3 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -20,6 +20,7 @@ package api
import (
"bytes"
"encoding/json"
+ "github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/mocks"
"github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,7 +34,32 @@ import (
"testing"
)
-func TestProcessScope(t *testing.T) {
+var bs = &core.BlueprintScopeV100{
+ Entities: []string{"CODE"},
+ Options: json.RawMessage(`{
+ "owner": "test",
+ "repo": "testRepo"
+ }`),
+ Transformation: json.RawMessage(`{
+ "prType": "hey,man,wasup",
+ "refdiff": {
+ "tagsPattern": "pattern",
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver"
+ },
+ "productionPattern": "xxxx"
+ }`),
+}
+
+var repo = &tasks.GithubApiRepo{
+ GithubId: 12345,
+ CloneUrl: "https://this_is_cloneUrl";,
+ CreatedAt: helper.Iso8601Time{},
+}
+
+func TestMakePipelinePlan(t *testing.T) {
+ prepareMockMeta(t)
+ mockApiClient := prepareMockClient(t, repo)
connection := &models.GithubConnection{
RestConnection: helper.RestConnection{
BaseConnection: helper.BaseConnection{
@@ -50,40 +76,9 @@ func TestProcessScope(t *testing.T) {
Token: "123",
},
}
- mockApiCLient := mocks.NewApiClientGetter(t)
- repo := &tasks.GithubApiRepo{
- GithubId: 12345,
- CloneUrl: "https://this_is_cloneUrl";,
- }
- js, err := json.Marshal(repo)
- assert.Nil(t, err)
- res := &http.Response{}
- res.Body = io.NopCloser(bytes.NewBuffer(js))
- res.StatusCode = http.StatusOK
- mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything,
mock.Anything).Return(res, nil)
- mockMeta := mocks.NewPluginMeta(t)
-
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
- err = core.RegisterPlugin("github", mockMeta)
- assert.Nil(t, err)
- bs := &core.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "owner": "test",
- "repo": "testRepo"
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
+ plan, err := makePipelinePlan(nil, scopes, mockApiClient, connection)
assert.Nil(t, err)
expectPlan := core.PipelinePlan{
@@ -134,3 +129,52 @@ func TestProcessScope(t *testing.T) {
}
assert.Equal(t, expectPlan, plan)
}
+
+func TestMemorizedGetApiRepo(t *testing.T) {
+ op := prepareOptions(t, bs)
+ expect := repo
+ repo1, err := memorizedGetApiRepo(repo, op, nil)
+ assert.Nil(t, err)
+ assert.Equal(t, expect, repo1)
+ mockApiClient := prepareMockClient(t, repo)
+ repo2, err := memorizedGetApiRepo(nil, op, mockApiClient)
+ assert.Nil(t, err)
+ assert.NotEqual(t, expect, repo2)
+}
+
+func TestGetApiRepo(t *testing.T) {
+ op := prepareOptions(t, bs)
+ mockClient := prepareMockClient(t, repo)
+ repo1, err := getApiRepo(op, mockClient)
+ assert.Nil(t, err)
+ assert.Equal(t, repo.GithubId, repo1.GithubId)
+}
+
+func prepareMockMeta(t *testing.T) {
+ mockMeta := mocks.NewPluginMeta(t)
+
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
+ err := core.RegisterPlugin("github", mockMeta)
+ assert.Nil(t, err)
+}
+
+func prepareMockClient(t *testing.T, repo *tasks.GithubApiRepo)
*mocks.ApiClientGetter {
+ mockApiCLient := mocks.NewApiClientGetter(t)
+ js, err := json.Marshal(repo)
+ assert.Nil(t, err)
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+ mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything,
mock.Anything).Return(res, nil)
+ return mockApiCLient
+}
+
+func prepareOptions(t *testing.T, bs *core.BlueprintScopeV100)
*tasks.GithubOptions {
+ options := make(map[string]interface{})
+ err := errors.Convert(json.Unmarshal(bs.Options, &options))
+ assert.Nil(t, err)
+ options["connectionId"] = 1
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ assert.Nil(t, err)
+ return op
+}
diff --git a/plugins/github/api/blueprint_v200.go
b/plugins/github/api/blueprint_v200.go
new file mode 100644
index 000000000..fdc320d0d
--- /dev/null
+++ b/plugins/github/api/blueprint_v200.go
@@ -0,0 +1,210 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/json"
+ goerror "errors"
+ "fmt"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/utils"
+ "github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
+ "gorm.io/gorm"
+ "strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+func MakeDataSourcePipelinePlanV200(subtaskMetas []core.SubTaskMeta,
connectionId uint64, bpScopes []*core.BlueprintScopeV200) (core.PipelinePlan,
[]core.Scope, errors.Error) {
+ db := basicRes.GetDal()
+ connectionHelper := helper.NewConnectionHelper(basicRes,
validator.New())
+ // get the connection info for url
+ connection := &models.GithubConnection{}
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ token := strings.Split(connection.Token, ",")[0]
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return nil, nil, err
+ }
+
+ plan := make(core.PipelinePlan, 0, len(bpScopes))
+ scopes := make([]core.Scope, 0, len(bpScopes))
+ for i, bpScope := range bpScopes {
+ var githubRepo *models.GithubRepo
+ // get repo from db
+ err = db.First(githubRepo, dal.Where(`id = ?`, bpScope.Id))
+ if err != nil {
+ return nil, nil, err
+ }
+ var transformationRule *models.TransformationRules
+ // get transformation rules from db
+ err = db.First(transformationRule, dal.Where(`id = ?`,
githubRepo.TransformationRuleId))
+ if err != nil && goerror.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil, err
+ }
+ var scope []core.Scope
+ // make pipeline for each bpScope
+ plan, scope, err = makeDataSourcePipelinePlanV200(subtaskMetas,
i, plan, bpScope, connection, apiClient, githubRepo, transformationRule)
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(scope) > 0 {
+ scopes = append(scopes, scope...)
+ }
+
+ }
+
+ return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+ subtaskMetas []core.SubTaskMeta,
+ i int,
+ plan core.PipelinePlan,
+ bpScope *core.BlueprintScopeV200,
+ connection *models.GithubConnection,
+ apiClient helper.ApiClientGetter,
+ githubRepo *models.GithubRepo,
+ transformationRule *models.TransformationRules,
+) (core.PipelinePlan, []core.Scope, errors.Error) {
+ var err errors.Error
+ var stage core.PipelineStage
+ var repo *tasks.GithubApiRepo
+ scopes := make([]core.Scope, 0)
+ // refdiff
+ if transformationRule != nil && transformationRule.ReffdiffRule != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ refdiffOp := map[string]interface{}{}
+ err =
errors.Convert(json.Unmarshal(transformationRule.ReffdiffRule, &refdiffOp))
+ if err != nil {
+ return nil, nil, err
+ }
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffOp,
+ },
+ }
+ }
+
+ // construct task options for github
+ var options map[string]interface{}
+ err = errors.Convert(mapstructure.Decode(githubRepo, &options))
+ if err != nil {
+ return nil, nil, err
+ }
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var transformationRuleMap map[string]interface{}
+ err = errors.Convert(mapstructure.Decode(transformationRule,
&transformationRuleMap))
+ if err != nil {
+ return nil, nil, err
+ }
+ options["transformationRules"] = transformationRuleMap
+ stage, err = addGithub(subtaskMetas, connection, bpScope.Entities,
stage, options)
+ if err != nil {
+ return nil, nil, err
+ }
+ // add gitex stage and add repo to scopes
+ if utils.StringsContains(bpScope.Entities, core.DOMAIN_TYPE_CODE) {
+ repo, err = memorizedGetApiRepo(repo, op, apiClient)
+ if err != nil {
+ return nil, nil, err
+ }
+ stage, err = addGitex(bpScope.Entities, connection, repo, stage)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopeRepo := &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
+ },
+ Name: fmt.Sprintf("%s/%s", githubRepo.OwnerLogin,
githubRepo.Name),
+ }
+ if repo.Parent != nil {
+ scopeRepo.ForkedFrom = repo.Parent.HTMLUrl
+ }
+ scopes = append(scopes, scopeRepo)
+ } else if utils.StringsContains(bpScope.Entities,
core.DOMAIN_TYPE_CODE_REVIEW) {
+ // if we don't need to collect gitex, we need to add repo to
scopes here
+ scopeRepo := &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
+ },
+ Name: fmt.Sprintf("%s/%s", githubRepo.OwnerLogin,
githubRepo.Name),
+ }
+ scopes = append(scopes, scopeRepo)
+ }
+ // add cicd_scope to scopes
+ if utils.StringsContains(bpScope.Entities, core.DOMAIN_TYPE_CICD) {
+ scopeCICD := &devops.CicdScope{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
+ },
+ Name: fmt.Sprintf("%s/%s", githubRepo.OwnerLogin,
githubRepo.Name),
+ }
+ scopes = append(scopes, scopeCICD)
+ }
+ // add board to scopes
+ if utils.StringsContains(bpScope.Entities, core.DOMAIN_TYPE_TICKET) {
+ scopeTicket := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
+ },
+ Name: fmt.Sprintf("%s/%s", githubRepo.OwnerLogin,
githubRepo.Name),
+ }
+ scopes = append(scopes, scopeTicket)
+ }
+
+ plan[i] = stage
+
+ return plan, scopes, nil
+}
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index e8281794d..ae869d2ce 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -245,6 +245,10 @@ func (plugin Github) MakePipelinePlan(connectionId uint64,
scope []*core.Bluepri
return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
}
+func (plugin Github) MakeDataSourcePipelinePlanV200(connectionId uint64,
scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err
errors.Error) {
+ return api.MakeDataSourcePipelinePlanV200(plugin.SubTaskMetas(),
connectionId, scopes)
+}
+
func (plugin Github) Close(taskCtx core.TaskContext) errors.Error {
data, ok := taskCtx.GetData().(*tasks.GithubTaskData)
if !ok {
diff --git a/plugins/github/models/repo.go b/plugins/github/models/repo.go
index 5ae5516ba..58613375b 100644
--- a/plugins/github/models/repo.go
+++ b/plugins/github/models/repo.go
@@ -23,20 +23,20 @@ import (
)
type GithubRepo struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
- TransformationRuleId uint64
- Name string `gorm:"type:varchar(255)"`
- HTMLUrl string `gorm:"type:varchar(255)"`
- Description string
- OwnerId int `json:"ownerId"`
- OwnerLogin string `json:"ownerLogin"
gorm:"type:varchar(255)"`
- Language string `json:"language"
gorm:"type:varchar(255)"`
- ParentGithubId int `json:"parentId"`
- ParentHTMLUrl string `json:"parentHtmlUrl"`
- CreatedDate time.Time `json:"createdDate"`
- UpdatedDate *time.Time `json:"updatedDate"`
- common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"
mapstructure:"connectionId,omitempty"`
+ GithubId int `gorm:"primaryKey" mapstructure:"-"`
+ Name string `gorm:"type:varchar(255)"
mapstructure:"repo,omitempty"`
+ HTMLUrl string `gorm:"type:varchar(255)"
mapstructure:"htmlUrl,omitempty"`
+ Description string `mapstructure:"description,omitempty"`
+ TransformationRuleId uint64
`mapstructure:"transformationRules,omitempty"`
+ OwnerId int `json:"ownerId"
mapstructure:"ownerId,omitempty"`
+ OwnerLogin string `json:"ownerLogin"
gorm:"type:varchar(255)" mapstructure:"owner,omitempty"`
+ Language string `json:"language"
gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
+ ParentGithubId int `json:"parentId"
mapstructure:"parentId,omitempty"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"
mapstructure:"parentHtmlUrl,omitempty"`
+ CreatedDate time.Time `json:"createdDate" mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ common.NoPKModel `mapstructure:"-"`
}
func (GithubRepo) TableName() string {
diff --git a/plugins/github/models/transformation_rule.go
b/plugins/github/models/transformation_rule.go
index f53eece58..16af8bb80 100644
--- a/plugins/github/models/transformation_rule.go
+++ b/plugins/github/models/transformation_rule.go
@@ -24,18 +24,18 @@ import (
)
type TransformationRules struct {
- common.Model
- PrType string `mapstructure:"prType"
json:"prType" gorm:"type:varchar(255)"`
- PrComponent string `mapstructure:"prComponent"
json:"prComponent" gorm:"type:varchar(255)"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern" gorm:"type:varchar(255)"`
- IssueSeverity string `mapstructure:"issueSeverity"
json:"issueSeverity" gorm:"type:varchar(255)"`
- IssuePriority string `mapstructure:"issuePriority"
json:"issuePriority" gorm:"type:varchar(255)"`
- IssueComponent string `mapstructure:"issueComponent"
json:"issueComponent" gorm:"type:varchar(255)"`
- IssueTypeBug string `mapstructure:"issueTypeBug"
json:"issueTypeBug" gorm:"type:varchar(255)"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident"
json:"issueTypeIncident" gorm:"type:varchar(255)"`
- IssueTypeRequirement string
`mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"
gorm:"type:varchar(255)"`
- DeploymentPattern string `mapstructure:"deploymentPattern"
json:"deploymentPattern" gorm:"type:varchar(255)"`
- RefdiffRule json.RawMessage `mapstructure:"refdiffRule"
json:"refdiffRule"`
+ common.Model `mapstructure:"-"`
+ PrType string `mapstructure:"prType,omitempty"
json:"prType" gorm:"type:varchar(255)"`
+ PrComponent string
`mapstructure:"prComponent,omitempty" json:"prComponent"
gorm:"type:varchar(255)"`
+ PrBodyClosePattern string
`mapstructure:"prBodyClosePattern,omitempty" json:"prBodyClosePattern"
gorm:"type:varchar(255)"`
+ IssueSeverity string
`mapstructure:"issueSeverity,omitempty" json:"issueSeverity"
gorm:"type:varchar(255)"`
+ IssuePriority string
`mapstructure:"issuePriority,omitempty" json:"issuePriority"
gorm:"type:varchar(255)"`
+ IssueComponent string
`mapstructure:"issueComponent,omitempty" json:"issueComponent"
gorm:"type:varchar(255)"`
+ IssueTypeBug string
`mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug"
gorm:"type:varchar(255)"`
+ IssueTypeIncident string
`mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident"
gorm:"type:varchar(255)"`
+ IssueTypeRequirement string
`mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement"
gorm:"type:varchar(255)"`
+ DeploymentPattern string
`mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"
gorm:"type:varchar(255)"`
+ ReffdiffRule json.RawMessage `mapstructure:"-" json:"refdiff"`
}
func (TransformationRules) TableName() string {
