This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 83b457eac feat(gitlab,bitbucket): warn when adding a repo already
scoped on another connection (#9047)
83b457eac is described below
commit 83b457eacd8a121367422d3d97fb13ea24b00e56
Author: Francisc Munteanu <[email protected]>
AuthorDate: Fri Aug 14 09:38:21 2026 +0200
feat(gitlab,bitbucket): warn when adding a repo already scoped on another
connection (#9047)
* feat(gitlab,bitbucket): extend scope-duplicate warning to GitLab and
Bitbucket
Add scope-duplicates API endpoints for GitLab and Bitbucket, mirroring the
existing GitHub implementation, and generalize the config-ui
data-scope-remote
component and scope API client so the duplicate-connection warning works for
all three plugins.
Co-Authored-By: Cursor Agent <[email protected]>
Co-authored-by: Cursor <[email protected]>
* assert specific error message
---------
Co-authored-by: Cursor Agent <[email protected]>
Co-authored-by: Cursor <[email protected]>
---
.../plugins/bitbucket/api/scope_duplicates_api.go | 200 ++++++++++++++++++++
.../bitbucket/api/scope_duplicates_api_test.go | 114 ++++++++++++
backend/plugins/bitbucket/impl/impl.go | 3 +
backend/plugins/gitlab/api/scope_duplicates_api.go | 205 +++++++++++++++++++++
.../gitlab/api/scope_duplicates_api_test.go | 114 ++++++++++++
backend/plugins/gitlab/impl/impl.go | 3 +
config-ui/src/api/scope/index.ts | 16 +-
.../data-scope-remote/data-scope-remote.tsx | 55 +++---
8 files changed, 684 insertions(+), 26 deletions(-)
diff --git a/backend/plugins/bitbucket/api/scope_duplicates_api.go
b/backend/plugins/bitbucket/api/scope_duplicates_api.go
new file mode 100644
index 000000000..0b7e998dd
--- /dev/null
+++ b/backend/plugins/bitbucket/api/scope_duplicates_api.go
@@ -0,0 +1,200 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// ScopeDuplicateConnection is a connection that shares a repository scope.
+type ScopeDuplicateConnection struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ConnectionName string `json:"connectionName"`
+}
+
+// ScopeDuplicateGroup is one repository that appears under multiple
connections
+// (diagnostics) or already exists under another connection (pre-add check).
+// Unlike GitHub/GitLab, a Bitbucket repo's natural id (BitbucketId) is the
+// "owner/repo" full name string rather than a numeric id.
+type ScopeDuplicateGroup struct {
+ BitbucketId string `json:"bitbucketId"`
+ HTMLUrl string `json:"htmlUrl"`
+ FullName string `json:"fullName"`
+ Connections []ScopeDuplicateConnection `json:"connections"`
+}
+
+// ScopeDuplicatesOutput is the response body for GetScopeDuplicates.
+type ScopeDuplicatesOutput struct {
+ Duplicates []ScopeDuplicateGroup `json:"duplicates"`
+}
+
+// scopeDuplicateRow is one joined row from the scoped SQL query.
+type scopeDuplicateRow struct {
+ BitbucketId string `gorm:"column:bitbucket_id"`
+ HTMLUrl string `gorm:"column:html_url"`
+ ConnectionId uint64 `gorm:"column:connection_id"`
+ ConnectionName string `gorm:"column:connection_name"`
+}
+
+// GetScopeDuplicates returns Bitbucket repositories registered under more than
+// one connection, or (with connectionId + bitbucketIds) candidates already
+// present on other connections.
+// @Summary Find Bitbucket scopes duplicated across connections
+// @Description Diagnostics: groups where the same bitbucketId (owner/repo)
appears on more than one connection.
+// @Description Pre-add check: pass connectionId and bitbucketIds to find
candidates already registered elsewhere.
+// @Tags plugins/bitbucket
+// @Param connectionId query int false "Current connection id (pre-add check)"
+// @Param bitbucketIds query string false "Comma-separated Bitbucket repo full
names (owner/repo) to check (pre-add check)"
+// @Success 200 {object} ScopeDuplicatesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/scope-duplicates [GET]
+func GetScopeDuplicates(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, bitbucketIds, err := parseScopeDuplicateQuery(input)
+ if err != nil {
+ return nil, err
+ }
+
+ // Pre-add check with an empty selection: nothing to warn about.
+ if connectionId != nil && len(bitbucketIds) == 0 {
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
[]ScopeDuplicateGroup{}},
+ Status: http.StatusOK,
+ }, nil
+ }
+
+ rows, err := queryScopeDuplicateRows(basicRes.GetDal(), connectionId,
bitbucketIds)
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
groupScopeDuplicateRows(rows)},
+ Status: http.StatusOK,
+ }, nil
+}
+
+func parseScopeDuplicateQuery(input *plugin.ApiResourceInput) (*uint64,
[]string, errors.Error) {
+ var connectionId *uint64
+ if v := input.Query.Get("connectionId"); v != "" {
+ id, err := strconv.ParseUint(v, 10, 64)
+ if err != nil {
+ return nil, nil, errors.BadInput.Wrap(err, "invalid
connectionId")
+ }
+ connectionId = &id
+ }
+
+ var bitbucketIds []string
+ if v := input.Query.Get("bitbucketIds"); v != "" {
+ for _, part := range strings.Split(v, ",") {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ bitbucketIds = append(bitbucketIds, part)
+ }
+ }
+
+ if len(bitbucketIds) > 0 && connectionId == nil {
+ return nil, nil, errors.BadInput.New("connectionId is required
when bitbucketIds is provided")
+ }
+
+ return connectionId, bitbucketIds, nil
+}
+
+// queryScopeDuplicateRows loads only the rows needed for the requested mode.
+// Check mode: selected bitbucketIds on any connection other than connectionId.
+// Diagnostics: bitbucketIds that already appear on more than one connection.
+func queryScopeDuplicateRows(db dal.Dal, connectionId *uint64, bitbucketIds
[]string) ([]scopeDuplicateRow, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("r.bitbucket_id, r.html_url, r.connection_id, c.name
AS connection_name"),
+ dal.From("_tool_bitbucket_repos r"),
+ dal.Join("INNER JOIN _tool_bitbucket_connections c ON c.id =
r.connection_id"),
+ dal.Orderby("r.bitbucket_id ASC, r.connection_id ASC"),
+ }
+
+ if connectionId != nil {
+ clauses = append(clauses, dal.Where(
+ "r.bitbucket_id IN ? AND r.connection_id != ?",
+ bitbucketIds,
+ *connectionId,
+ ))
+ } else {
+ clauses = append(clauses, dal.Where(`r.bitbucket_id IN (
+ SELECT bitbucket_id FROM _tool_bitbucket_repos
+ GROUP BY bitbucket_id
+ HAVING COUNT(DISTINCT connection_id) > 1
+ )`))
+ }
+
+ var rows []scopeDuplicateRow
+ if err := db.All(&rows, clauses...); err != nil {
+ return nil, err
+ }
+ return rows, nil
+}
+
+// groupScopeDuplicateRows collapses already-filtered SQL rows into API groups.
+func groupScopeDuplicateRows(rows []scopeDuplicateRow) []ScopeDuplicateGroup {
+ if len(rows) == 0 {
+ return []ScopeDuplicateGroup{}
+ }
+
+ result := make([]ScopeDuplicateGroup, 0)
+ var current *ScopeDuplicateGroup
+ seenConns := make(map[uint64]struct{})
+
+ flush := func() {
+ if current != nil {
+ result = append(result, *current)
+ }
+ }
+
+ for _, row := range rows {
+ if current == nil || current.BitbucketId != row.BitbucketId {
+ flush()
+ current = &ScopeDuplicateGroup{
+ BitbucketId: row.BitbucketId,
+ HTMLUrl: row.HTMLUrl,
+ // Bitbucket's "full name" is the owner/repo id
itself, see BitbucketRepo.ScopeFullName().
+ FullName: row.BitbucketId,
+ Connections: make([]ScopeDuplicateConnection,
0, 2),
+ }
+ seenConns = make(map[uint64]struct{})
+ }
+ if current.HTMLUrl == "" && row.HTMLUrl != "" {
+ current.HTMLUrl = row.HTMLUrl
+ }
+ if _, ok := seenConns[row.ConnectionId]; ok {
+ continue
+ }
+ seenConns[row.ConnectionId] = struct{}{}
+ current.Connections = append(current.Connections,
ScopeDuplicateConnection{
+ ConnectionId: row.ConnectionId,
+ ConnectionName: row.ConnectionName,
+ })
+ }
+ flush()
+ return result
+}
diff --git a/backend/plugins/bitbucket/api/scope_duplicates_api_test.go
b/backend/plugins/bitbucket/api/scope_duplicates_api_test.go
new file mode 100644
index 000000000..13a31b493
--- /dev/null
+++ b/backend/plugins/bitbucket/api/scope_duplicates_api_test.go
@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/url"
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGroupScopeDuplicateRows_Empty(t *testing.T) {
+ assert.Empty(t, groupScopeDuplicateRows(nil))
+ assert.Empty(t, groupScopeDuplicateRows([]scopeDuplicateRow{}))
+}
+
+func TestGroupScopeDuplicateRows_GroupsConnections(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {BitbucketId: "o/a", HTMLUrl: "https://bitbucket.org/o/a";,
ConnectionId: 1, ConnectionName: "Bitbucket Production"},
+ {BitbucketId: "o/a", HTMLUrl: "https://bitbucket.org/o/a";,
ConnectionId: 2, ConnectionName: "Bitbucket Staging"},
+ {BitbucketId: "o/b", HTMLUrl: "https://bitbucket.org/o/b";,
ConnectionId: 3, ConnectionName: "Other"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Equal(t, []ScopeDuplicateGroup{
+ {
+ BitbucketId: "o/a",
+ HTMLUrl: "https://bitbucket.org/o/a";,
+ FullName: "o/a",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "Bitbucket
Production"},
+ {ConnectionId: 2, ConnectionName: "Bitbucket
Staging"},
+ },
+ },
+ {
+ BitbucketId: "o/b",
+ HTMLUrl: "https://bitbucket.org/o/b";,
+ FullName: "o/b",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 3, ConnectionName: "Other"},
+ },
+ },
+ }, got)
+}
+
+func TestGroupScopeDuplicateRows_DedupesSameConnection(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {BitbucketId: "o/a", ConnectionId: 1, ConnectionName: "Prod"},
+ {BitbucketId: "o/a", ConnectionId: 1, ConnectionName: "Prod"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "Prod"},
+ }, got[0].Connections)
+}
+
+func TestGroupScopeDuplicateRows_FillsMissingHTMLUrl(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {BitbucketId: "o/a", ConnectionId: 1, ConnectionName: "A"},
+ {BitbucketId: "o/a", HTMLUrl: "https://bitbucket.org/o/a";,
ConnectionId: 2, ConnectionName: "B"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, "https://bitbucket.org/o/a";, got[0].HTMLUrl)
+ assert.Equal(t, "o/a", got[0].FullName)
+}
+
+func TestParseScopeDuplicateQuery(t *testing.T) {
+ input := &plugin.ApiResourceInput{Query: url.Values{}}
+ connId, ids, err := parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Nil(t, connId)
+ assert.Empty(t, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"3"},
+ "bitbucketIds": []string{"o/a, o/b,o/c"},
+ }}
+ connId, ids, err = parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Equal(t, uint64(3), *connId)
+ assert.Equal(t, []string{"o/a", "o/b", "o/c"}, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "bitbucketIds": []string{"o/a"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Contains(t, err.Error(), "connectionId is required when
bitbucketIds is provided")
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"abc"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Contains(t, err.Error(), "invalid connectionId")
+}
diff --git a/backend/plugins/bitbucket/impl/impl.go
b/backend/plugins/bitbucket/impl/impl.go
index 9586294ec..051e9b783 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -243,6 +243,9 @@ func (p Bitbucket) ApiResources()
map[string]map[string]plugin.ApiResourceHandle
"scope-config/:scopeConfigId/projects": {
"GET": api.GetProjectsByScopeConfig,
},
+ "scope-duplicates": {
+ "GET": api.GetScopeDuplicates,
+ },
}
}
diff --git a/backend/plugins/gitlab/api/scope_duplicates_api.go
b/backend/plugins/gitlab/api/scope_duplicates_api.go
new file mode 100644
index 000000000..dd0517d02
--- /dev/null
+++ b/backend/plugins/gitlab/api/scope_duplicates_api.go
@@ -0,0 +1,205 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// ScopeDuplicateConnection is a connection that shares a project scope.
+type ScopeDuplicateConnection struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ConnectionName string `json:"connectionName"`
+}
+
+// ScopeDuplicateGroup is one project that appears under multiple connections
+// (diagnostics) or already exists under another connection (pre-add check).
+type ScopeDuplicateGroup struct {
+ GitlabId int `json:"gitlabId"`
+ HTMLUrl string `json:"htmlUrl"`
+ FullName string `json:"fullName"`
+ Connections []ScopeDuplicateConnection `json:"connections"`
+}
+
+// ScopeDuplicatesOutput is the response body for GetScopeDuplicates.
+type ScopeDuplicatesOutput struct {
+ Duplicates []ScopeDuplicateGroup `json:"duplicates"`
+}
+
+// scopeDuplicateRow is one joined row from the scoped SQL query.
+type scopeDuplicateRow struct {
+ GitlabId int `gorm:"column:gitlab_id"`
+ HTMLUrl string `gorm:"column:html_url"`
+ FullName string `gorm:"column:full_name"`
+ ConnectionId uint64 `gorm:"column:connection_id"`
+ ConnectionName string `gorm:"column:connection_name"`
+}
+
+// GetScopeDuplicates returns GitLab projects registered under more than one
+// connection, or (with connectionId + gitlabIds) candidates already present on
+// other connections.
+// @Summary Find GitLab scopes duplicated across connections
+// @Description Diagnostics: groups where the same gitlabId appears on more
than one connection.
+// @Description Pre-add check: pass connectionId and gitlabIds to find
candidates already registered elsewhere.
+// @Tags plugins/gitlab
+// @Param connectionId query int false "Current connection id (pre-add check)"
+// @Param gitlabIds query string false "Comma-separated GitLab project ids to
check (pre-add check)"
+// @Success 200 {object} ScopeDuplicatesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/scope-duplicates [GET]
+func GetScopeDuplicates(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, gitlabIds, err := parseScopeDuplicateQuery(input)
+ if err != nil {
+ return nil, err
+ }
+
+ // Pre-add check with an empty selection: nothing to warn about.
+ if connectionId != nil && len(gitlabIds) == 0 {
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
[]ScopeDuplicateGroup{}},
+ Status: http.StatusOK,
+ }, nil
+ }
+
+ rows, err := queryScopeDuplicateRows(basicRes.GetDal(), connectionId,
gitlabIds)
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
groupScopeDuplicateRows(rows)},
+ Status: http.StatusOK,
+ }, nil
+}
+
+func parseScopeDuplicateQuery(input *plugin.ApiResourceInput) (*uint64, []int,
errors.Error) {
+ var connectionId *uint64
+ if v := input.Query.Get("connectionId"); v != "" {
+ id, err := strconv.ParseUint(v, 10, 64)
+ if err != nil {
+ return nil, nil, errors.BadInput.Wrap(err, "invalid
connectionId")
+ }
+ connectionId = &id
+ }
+
+ var gitlabIds []int
+ if v := input.Query.Get("gitlabIds"); v != "" {
+ for _, part := range strings.Split(v, ",") {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ id, err := strconv.Atoi(part)
+ if err != nil {
+ return nil, nil, errors.BadInput.Wrap(err,
"invalid gitlabIds")
+ }
+ gitlabIds = append(gitlabIds, id)
+ }
+ }
+
+ if len(gitlabIds) > 0 && connectionId == nil {
+ return nil, nil, errors.BadInput.New("connectionId is required
when gitlabIds is provided")
+ }
+
+ return connectionId, gitlabIds, nil
+}
+
+// queryScopeDuplicateRows loads only the rows needed for the requested mode.
+// Check mode: selected gitlabIds on any connection other than connectionId.
+// Diagnostics: gitlabIds that already appear on more than one connection.
+func queryScopeDuplicateRows(db dal.Dal, connectionId *uint64, gitlabIds
[]int) ([]scopeDuplicateRow, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("r.gitlab_id, r.web_url AS html_url,
r.path_with_namespace AS full_name, r.connection_id, c.name AS
connection_name"),
+ dal.From("_tool_gitlab_projects r"),
+ dal.Join("INNER JOIN _tool_gitlab_connections c ON c.id =
r.connection_id"),
+ dal.Orderby("r.gitlab_id ASC, r.connection_id ASC"),
+ }
+
+ if connectionId != nil {
+ clauses = append(clauses, dal.Where(
+ "r.gitlab_id IN ? AND r.connection_id != ?",
+ gitlabIds,
+ *connectionId,
+ ))
+ } else {
+ clauses = append(clauses, dal.Where(`r.gitlab_id IN (
+ SELECT gitlab_id FROM _tool_gitlab_projects
+ GROUP BY gitlab_id
+ HAVING COUNT(DISTINCT connection_id) > 1
+ )`))
+ }
+
+ var rows []scopeDuplicateRow
+ if err := db.All(&rows, clauses...); err != nil {
+ return nil, err
+ }
+ return rows, nil
+}
+
+// groupScopeDuplicateRows collapses already-filtered SQL rows into API groups.
+func groupScopeDuplicateRows(rows []scopeDuplicateRow) []ScopeDuplicateGroup {
+ if len(rows) == 0 {
+ return []ScopeDuplicateGroup{}
+ }
+
+ result := make([]ScopeDuplicateGroup, 0)
+ var current *ScopeDuplicateGroup
+ seenConns := make(map[uint64]struct{})
+
+ flush := func() {
+ if current != nil {
+ result = append(result, *current)
+ }
+ }
+
+ for _, row := range rows {
+ if current == nil || current.GitlabId != row.GitlabId {
+ flush()
+ current = &ScopeDuplicateGroup{
+ GitlabId: row.GitlabId,
+ HTMLUrl: row.HTMLUrl,
+ FullName: row.FullName,
+ Connections: make([]ScopeDuplicateConnection,
0, 2),
+ }
+ seenConns = make(map[uint64]struct{})
+ }
+ if current.HTMLUrl == "" && row.HTMLUrl != "" {
+ current.HTMLUrl = row.HTMLUrl
+ }
+ if current.FullName == "" && row.FullName != "" {
+ current.FullName = row.FullName
+ }
+ if _, ok := seenConns[row.ConnectionId]; ok {
+ continue
+ }
+ seenConns[row.ConnectionId] = struct{}{}
+ current.Connections = append(current.Connections,
ScopeDuplicateConnection{
+ ConnectionId: row.ConnectionId,
+ ConnectionName: row.ConnectionName,
+ })
+ }
+ flush()
+ return result
+}
diff --git a/backend/plugins/gitlab/api/scope_duplicates_api_test.go
b/backend/plugins/gitlab/api/scope_duplicates_api_test.go
new file mode 100644
index 000000000..dd4d8d510
--- /dev/null
+++ b/backend/plugins/gitlab/api/scope_duplicates_api_test.go
@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/url"
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGroupScopeDuplicateRows_Empty(t *testing.T) {
+ assert.Empty(t, groupScopeDuplicateRows(nil))
+ assert.Empty(t, groupScopeDuplicateRows([]scopeDuplicateRow{}))
+}
+
+func TestGroupScopeDuplicateRows_GroupsConnections(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GitlabId: 100, HTMLUrl: "https://gitlab.com/o/a";, FullName:
"o/a", ConnectionId: 1, ConnectionName: "GitLab Production"},
+ {GitlabId: 100, HTMLUrl: "https://gitlab.com/o/a";, FullName:
"o/a", ConnectionId: 2, ConnectionName: "GitLab Staging"},
+ {GitlabId: 200, HTMLUrl: "https://gitlab.com/o/b";, FullName:
"o/b", ConnectionId: 3, ConnectionName: "Other"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Equal(t, []ScopeDuplicateGroup{
+ {
+ GitlabId: 100,
+ HTMLUrl: "https://gitlab.com/o/a";,
+ FullName: "o/a",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "GitLab
Production"},
+ {ConnectionId: 2, ConnectionName: "GitLab
Staging"},
+ },
+ },
+ {
+ GitlabId: 200,
+ HTMLUrl: "https://gitlab.com/o/b";,
+ FullName: "o/b",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 3, ConnectionName: "Other"},
+ },
+ },
+ }, got)
+}
+
+func TestGroupScopeDuplicateRows_DedupesSameConnection(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GitlabId: 100, FullName: "o/a", ConnectionId: 1,
ConnectionName: "Prod"},
+ {GitlabId: 100, FullName: "o/a", ConnectionId: 1,
ConnectionName: "Prod"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "Prod"},
+ }, got[0].Connections)
+}
+
+func TestGroupScopeDuplicateRows_FillsMissingLabels(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GitlabId: 100, ConnectionId: 1, ConnectionName: "A"},
+ {GitlabId: 100, HTMLUrl: "https://gitlab.com/o/a";, FullName:
"o/a", ConnectionId: 2, ConnectionName: "B"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, "https://gitlab.com/o/a";, got[0].HTMLUrl)
+ assert.Equal(t, "o/a", got[0].FullName)
+}
+
+func TestParseScopeDuplicateQuery(t *testing.T) {
+ input := &plugin.ApiResourceInput{Query: url.Values{}}
+ connId, ids, err := parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Nil(t, connId)
+ assert.Empty(t, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"3"},
+ "gitlabIds": []string{"10, 20,30"},
+ }}
+ connId, ids, err = parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Equal(t, uint64(3), *connId)
+ assert.Equal(t, []int{10, 20, 30}, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "gitlabIds": []string{"10"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Contains(t, err.Error(), "connectionId is required when
gitlabIds is provided")
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"abc"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Contains(t, err.Error(), "invalid connectionId")
+}
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 39e245c43..3dbd36720 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -279,6 +279,9 @@ func (p Gitlab) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"scope-config/:scopeConfigId/projects": {
"GET": api.GetProjectsByScopeConfig,
},
+ "scope-duplicates": {
+ "GET": api.GetScopeDuplicates,
+ },
}
}
diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts
index 8996f4662..359e6b9c7 100644
--- a/config-ui/src/api/scope/index.ts
+++ b/config-ui/src/api/scope/index.ts
@@ -102,21 +102,27 @@ export type ScopeDuplicateConnection = {
connectionName: string;
};
+// The plugin-specific id field (githubId/gitlabId/bitbucketId) is
intentionally
+// omitted here since the UI only ever needs
`htmlUrl`/`fullName`/`connections`.
export type ScopeDuplicateGroup = {
- githubId: number;
htmlUrl: string;
fullName: string;
connections: ScopeDuplicateConnection[];
};
+// The query param that carries the comma-separated scope ids is named
differently per
+// plugin (githubIds/gitlabIds/bitbucketIds), so callers pass it in via
`idsParam`.
export const scopeDuplicates = (
plugin: string,
data?: {
connectionId?: ID;
- githubIds?: string;
+ idsParam?: string;
+ ids?: string;
},
-): Promise<{ duplicates: ScopeDuplicateGroup[] }> =>
- request(`/plugins/${plugin}/scope-duplicates`, {
+): Promise<{ duplicates: ScopeDuplicateGroup[] }> => {
+ const { idsParam, ids, ...rest } = data ?? {};
+ return request(`/plugins/${plugin}/scope-duplicates`, {
method: 'get',
- data,
+ data: idsParam && ids !== undefined ? { ...rest, [idsParam]: ids } : rest,
});
+};
diff --git
a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
index 7453cbedc..896629e3f 100644
--- a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
+++ b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
@@ -39,28 +39,38 @@ interface Props {
onSubmit?: (origin: any) => void;
}
-const getGithubId = (scope: any): number | undefined => {
- const fromData = scope?.data?.githubId;
+// Plugins that support the "warn on duplicate scope" check, and the
field/query-param
+// names their scope-duplicates API uses. Bitbucket's id is a string
("owner/repo"),
+// GitHub/GitLab's is numeric, but both work fine as strings on the wire.
+const SCOPE_DUPLICATE_FIELDS: Record<string, { dataField: string; queryParam:
string }> = {
+ github: { dataField: 'githubId', queryParam: 'githubIds' },
+ gitlab: { dataField: 'gitlabId', queryParam: 'gitlabIds' },
+ bitbucket: { dataField: 'bitbucketId', queryParam: 'bitbucketIds' },
+};
+
+const getScopeDuplicateId = (plugin: string, scope: any): string | undefined
=> {
+ const dataField = SCOPE_DUPLICATE_FIELDS[plugin]?.dataField;
+ const fromData = dataField ? scope?.data?.[dataField] : undefined;
if (typeof fromData === 'number' && fromData > 0) {
+ return String(fromData);
+ }
+ if (typeof fromData === 'string' && fromData) {
return fromData;
}
- const fromId = Number(scope?.id);
- if (!Number.isNaN(fromId) && fromId > 0) {
- return fromId;
+ if (scope?.id !== undefined && scope?.id !== null && String(scope.id) !==
'') {
+ return String(scope.id);
}
return undefined;
};
const buildDuplicateWarning = (duplicates: ScopeDuplicateGroup[]): string => {
const connectionNames = Array.from(
- new Set(
- duplicates.flatMap((d) => d.connections.map((c) =>
c.connectionName).filter(Boolean)),
- ),
+ new Set(duplicates.flatMap((d) => d.connections.map((c) =>
c.connectionName).filter(Boolean))),
);
const repoLabel =
duplicates.length === 1
- ? duplicates[0].fullName || duplicates[0].htmlUrl || 'This repository'
- : 'One or more selected repositories';
+ ? duplicates[0].fullName || duplicates[0].htmlUrl || 'This item'
+ : 'One or more selected items';
const via =
connectionNames.length === 1
@@ -69,7 +79,7 @@ const buildDuplicateWarning = (duplicates:
ScopeDuplicateGroup[]): string => {
? `Connections ${connectionNames.map((n) => `"${n}"`).join(', ')}`
: 'another connection';
- return `${repoLabel} is already connected via ${via}. Collecting it here
will create duplicate pull requests and issue records, which will inflate all
metrics for this repository.`;
+ return `${repoLabel} is already connected via ${via}. Collecting it here
will create duplicate records, which will inflate all metrics for this
repository.`;
};
export const DataScopeRemote = ({
@@ -94,19 +104,21 @@ export const DataScopeRemote = ({
const config = useMemo(() => getPluginConfig(plugin).dataScope, [plugin]);
- const githubIdsKey = useMemo(() => {
- if (plugin !== 'github') {
+ const duplicateFields = SCOPE_DUPLICATE_FIELDS[plugin];
+
+ const idsKey = useMemo(() => {
+ if (!duplicateFields) {
return '';
}
return selectedScope
- .map(getGithubId)
- .filter((id): id is number => id !== undefined)
- .sort((a, b) => a - b)
+ .map((it) => getScopeDuplicateId(plugin, it))
+ .filter((id): id is string => id !== undefined)
+ .sort()
.join(',');
- }, [plugin, selectedScope]);
+ }, [plugin, duplicateFields, selectedScope]);
useEffect(() => {
- if (plugin !== 'github' || !githubIdsKey) {
+ if (!duplicateFields || !idsKey) {
setDuplicates([]);
setWarningDismissed(false);
return;
@@ -118,7 +130,8 @@ export const DataScopeRemote = ({
API.scope
.scopeDuplicates(plugin, {
connectionId,
- githubIds: githubIdsKey,
+ idsParam: duplicateFields.queryParam,
+ ids: idsKey,
})
.then((res) => {
if (!cancelled) {
@@ -134,7 +147,7 @@ export const DataScopeRemote = ({
return () => {
cancelled = true;
};
- }, [plugin, connectionId, githubIdsKey]);
+ }, [plugin, connectionId, duplicateFields, idsKey]);
const handleSubmit = async () => {
const [success, res] = await operator(
@@ -150,7 +163,7 @@ export const DataScopeRemote = ({
}
};
- const showWarning = plugin === 'github' && duplicates.length > 0 &&
!warningDismissed;
+ const showWarning = !!duplicateFields && duplicates.length > 0 &&
!warningDismissed;
return (
<Flex vertical>
