This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 003e5d000 feat: support run bp v100 for jira & gitlab (#3957)
003e5d000 is described below
commit 003e5d000614a2953067a186f086310583058bb1
Author: Likyh <[email protected]>
AuthorDate: Fri Dec 16 18:08:47 2022 +0800
feat: support run bp v100 for jira & gitlab (#3957)
* feat: support run bp v100 for jira & gitlab
* fix: fix for lint
---
plugins/gitlab/api/blueprint_v200.go | 28 ++++++++++++-
plugins/gitlab/impl/impl.go | 26 ++++++++++++
...able.go => 20221125_add_html_url_to_project.go} | 3 +-
plugins/gitlab/tasks/project_extractor.go | 47 ++++++++++++++++++++++
plugins/jira/api/scope.go | 28 ++++++++++++-
plugins/jira/impl/impl.go | 33 +++++++++++++--
6 files changed, 157 insertions(+), 8 deletions(-)
diff --git a/plugins/gitlab/api/blueprint_v200.go
b/plugins/gitlab/api/blueprint_v200.go
index b3228fc6d..1ec1078d4 100644
--- a/plugins/gitlab/api/blueprint_v200.go
+++ b/plugins/gitlab/api/blueprint_v200.go
@@ -18,14 +18,15 @@ limitations under the License.
package api
import (
+ "encoding/json"
"fmt"
+ "io"
+ "net/http"
"net/url"
"strconv"
"time"
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/utils"
-
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -33,7 +34,9 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
)
func MakePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId
uint64, scope []*core.BlueprintScopeV200, syncPolicy *core.BlueprintSyncPolicy)
(core.PipelinePlan, []core.Scope, errors.Error) {
@@ -210,3 +213,24 @@ func GetTransformationRuleByRepo(repo
*models.GitlabProject) (*models.GitlabTran
return transformationRules, nil
}
+
+func GetApiProject(op *tasks.GitlabOptions, apiClient helper.ApiClientGetter)
(*tasks.GitlabApiProject, errors.Error) {
+ repoRes := &tasks.GitlabApiProject{}
+ res, err := apiClient.Get(fmt.Sprintf("projects/%d", op.ProjectId),
nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return nil, err
+ }
+ err = errors.Convert(json.Unmarshal(body, repoRes))
+ if err != nil {
+ return nil, err
+ }
+ return repoRes, nil
+}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 49f5738d0..e0eeb3834 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -19,11 +19,13 @@ package impl
import (
"fmt"
+ "gorm.io/gorm"
"strconv"
"time"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitlab/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts"
@@ -159,6 +161,30 @@ func (plugin Gitlab) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
}
}
+ if op.ProjectId != 0 {
+ var scope *models.GitlabProject
+ // support v100 & advance mode
+ // If we still cannot find the record in db, we have to request
from remote server and save it to db
+ err = taskCtx.GetDal().First(&scope, dal.Where("connection_id =
? AND gitlab_id = ?", op.ConnectionId, op.ProjectId))
+ if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
+ var project *tasks.GitlabApiProject
+ project, err = api.GetApiProject(op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ logger.Debug(fmt.Sprintf("Current project: %d",
project.GitlabId))
+ scope = tasks.ConvertProject(project)
+ scope.ConnectionId = op.ConnectionId
+ err = taskCtx.GetDal().CreateIfNotExist(&scope)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find project: %d", op.ProjectId))
+ }
+ }
+
if op.GitlabTransformationRule == nil && op.ProjectId != 0 {
repo, err :=
api.GetRepoByConnectionIdAndscopeId(op.ConnectionId, strconv.Itoa(op.ProjectId))
if err != nil {
diff --git
a/plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
b/plugins/gitlab/models/migrationscripts/20221125_add_html_url_to_project.go
similarity index 95%
rename from
plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
rename to
plugins/gitlab/models/migrationscripts/20221125_add_html_url_to_project.go
index b82fe4932..d51bd6d98 100644
---
a/plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
+++ b/plugins/gitlab/models/migrationscripts/20221125_add_html_url_to_project.go
@@ -26,6 +26,7 @@ import (
type gitlabProject20221125 struct {
TransformationRuleId uint64
+ HttpUrlToRepo string
}
func (gitlabProject20221125) TableName() string {
@@ -43,5 +44,5 @@ func (*addTransformationRule20221125) Version() uint64 {
}
func (*addTransformationRule20221125) Name() string {
- return "add table _tool_gitlab_transformation_rules, add
transformation_rule_id to _tool_gitlab_projects"
+ return "add table _tool_gitlab_transformation_rules, add
transformation_rule_id and html_url to _tool_gitlab_projects"
}
diff --git a/plugins/gitlab/tasks/project_extractor.go
b/plugins/gitlab/tasks/project_extractor.go
new file mode 100644
index 000000000..e873fe8eb
--- /dev/null
+++ b/plugins/gitlab/tasks/project_extractor.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+// Convert the API response to our DB model instance
+func ConvertProject(gitlabApiProject *GitlabApiProject) *models.GitlabProject {
+ gitlabProject := &models.GitlabProject{
+ GitlabId: gitlabApiProject.GitlabId,
+ Name: gitlabApiProject.Name,
+ Description: gitlabApiProject.Description,
+ DefaultBranch: gitlabApiProject.DefaultBranch,
+ CreatorId: gitlabApiProject.CreatorId,
+ PathWithNamespace: gitlabApiProject.PathWithNamespace,
+ WebUrl: gitlabApiProject.WebUrl,
+ HttpUrlToRepo: gitlabApiProject.HttpUrlToRepo,
+ Visibility: gitlabApiProject.Visibility,
+ OpenIssuesCount: gitlabApiProject.OpenIssuesCount,
+ StarCount: gitlabApiProject.StarCount,
+ CreatedDate: gitlabApiProject.CreatedAt.ToTime(),
+ UpdatedDate:
helper.Iso8601TimeToTime(gitlabApiProject.LastActivityAt),
+ }
+ if gitlabApiProject.ForkedFromProject != nil {
+ gitlabProject.ForkedFromProjectId =
gitlabApiProject.ForkedFromProject.GitlabId
+ gitlabProject.ForkedFromProjectWebUrl =
gitlabApiProject.ForkedFromProject.WebUrl
+ }
+ return gitlabProject
+}
diff --git a/plugins/jira/api/scope.go b/plugins/jira/api/scope.go
index 5de059776..ff4a406a9 100644
--- a/plugins/jira/api/scope.go
+++ b/plugins/jira/api/scope.go
@@ -18,6 +18,10 @@ limitations under the License.
package api
import (
+ "encoding/json"
+ "fmt"
+ "gorm.io/gorm"
+ "io"
"net/http"
"strconv"
@@ -26,8 +30,9 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jira/models"
+ "github.com/apache/incubator-devlake/plugins/jira/tasks"
+ "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
"github.com/mitchellh/mapstructure"
- "gorm.io/gorm"
)
type apiBoard struct {
@@ -211,3 +216,24 @@ func verifyBoard(board *models.JiraBoard) errors.Error {
}
return nil
}
+
+func GetApiJira(op *tasks.JiraOptions, apiClient helper.ApiClientGetter)
(*apiv2models.Board, errors.Error) {
+ boardRes := &apiv2models.Board{}
+ res, err := apiClient.Get(fmt.Sprintf("agile/1.0/board/%d",
op.BoardId), nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return nil, err
+ }
+ err = errors.Convert(json.Unmarshal(body, boardRes))
+ if err != nil {
+ return nil, err
+ }
+ return boardRes, nil
+}
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index 7a9cdf013..bd69cd76c 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -19,6 +19,7 @@ package impl
import (
"fmt"
+ "gorm.io/gorm"
"net/http"
"time"
@@ -30,6 +31,7 @@ import (
"github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/jira/tasks"
+ "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
var _ core.PluginMeta = (*Jira)(nil)
@@ -165,6 +167,33 @@ func (plugin Jira) PrepareTaskData(taskCtx
core.TaskContext, options map[string]
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get Jira
connection")
}
+ jiraApiClient, err := tasks.NewJiraApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "failed to create jira api
client")
+ }
+
+ if op.BoardId != 0 {
+ var scope *models.JiraBoard
+ // support v100 & advance mode
+ // If we still cannot find the record in db, we have to request
from remote server and save it to db
+ err = taskCtx.GetDal().First(&scope, dal.Where("connection_id =
? AND board_id = ?", op.ConnectionId, op.BoardId))
+ if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
+ var board *apiv2models.Board
+ board, err = api.GetApiJira(&op, jiraApiClient)
+ if err != nil {
+ return nil, err
+ }
+ logger.Debug(fmt.Sprintf("Current project: %d",
board.ID))
+ scope = board.ToToolLayer(connection.ID)
+ err = taskCtx.GetDal().CreateIfNotExist(&scope)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find board: %d", op.BoardId))
+ }
+ }
if op.BoardId == 0 && op.ScopeId != "" {
var jiraBoard models.JiraBoard
@@ -198,10 +227,6 @@ func (plugin Jira) PrepareTaskData(taskCtx
core.TaskContext, options map[string]
}
}
- jiraApiClient, err := tasks.NewJiraApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "failed to create jira api
client")
- }
info, code, err := tasks.GetJiraServerInfo(jiraApiClient)
if err != nil || code != http.StatusOK || info == nil {
return nil, errors.HttpStatus(code).Wrap(err, "fail to get Jira
server info")
