This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d323c1697 feat(github): warn when adding a repo already scoped on
another connection (#9034)
d323c1697 is described below
commit d323c1697d04c199150dfdc85f6280dfa6b14762
Author: Francisc Munteanu <[email protected]>
AuthorDate: Sat Aug 8 07:19:53 2026 +0200
feat(github): warn when adding a repo already scoped on another connection
(#9034)
Add a scoped GET /plugins/github/scope-duplicates API and a dismissible
Config-UI Alert so users are informed before creating cross-connection
duplicate domain records (#8947).
Co-authored-by: Cursor <[email protected]>
Co-authored-by: Cursor <[email protected]>
---
backend/plugins/github/api/scope_duplicates_api.go | 205 +++++++++++++++++++++
.../github/api/scope_duplicates_api_test.go | 114 ++++++++++++
backend/plugins/github/impl/impl.go | 3 +
config-ui/src/api/scope/index.ts | 24 +++
.../data-scope-remote/data-scope-remote.tsx | 92 ++++++++-
5 files changed, 437 insertions(+), 1 deletion(-)
diff --git a/backend/plugins/github/api/scope_duplicates_api.go
b/backend/plugins/github/api/scope_duplicates_api.go
new file mode 100644
index 000000000..e3cc2312b
--- /dev/null
+++ b/backend/plugins/github/api/scope_duplicates_api.go
@@ -0,0 +1,205 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// ScopeDuplicateConnection is a connection that shares a repository scope.
+type ScopeDuplicateConnection struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ConnectionName string `json:"connectionName"`
+}
+
+// ScopeDuplicateGroup is one repository that appears under multiple
connections
+// (diagnostics) or already exists under another connection (pre-add check).
+type ScopeDuplicateGroup struct {
+ GithubId int `json:"githubId"`
+ HTMLUrl string `json:"htmlUrl"`
+ FullName string `json:"fullName"`
+ Connections []ScopeDuplicateConnection `json:"connections"`
+}
+
+// ScopeDuplicatesOutput is the response body for GetScopeDuplicates.
+type ScopeDuplicatesOutput struct {
+ Duplicates []ScopeDuplicateGroup `json:"duplicates"`
+}
+
+// scopeDuplicateRow is one joined row from the scoped SQL query.
+type scopeDuplicateRow struct {
+ GithubId int `gorm:"column:github_id"`
+ HTMLUrl string `gorm:"column:html_url"`
+ FullName string `gorm:"column:full_name"`
+ ConnectionId uint64 `gorm:"column:connection_id"`
+ ConnectionName string `gorm:"column:connection_name"`
+}
+
+// GetScopeDuplicates returns GitHub repositories registered under more than
one
+// connection, or (with connectionId + githubIds) candidates already present on
+// other connections.
+// @Summary Find GitHub scopes duplicated across connections
+// @Description Diagnostics: groups where the same githubId appears on more
than one connection.
+// @Description Pre-add check: pass connectionId and githubIds to find
candidates already registered elsewhere.
+// @Tags plugins/github
+// @Param connectionId query int false "Current connection id (pre-add check)"
+// @Param githubIds query string false "Comma-separated GitHub repo ids to
check (pre-add check)"
+// @Success 200 {object} ScopeDuplicatesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/scope-duplicates [GET]
+func GetScopeDuplicates(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, githubIds, err := parseScopeDuplicateQuery(input)
+ if err != nil {
+ return nil, err
+ }
+
+ // Pre-add check with an empty selection: nothing to warn about.
+ if connectionId != nil && len(githubIds) == 0 {
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
[]ScopeDuplicateGroup{}},
+ Status: http.StatusOK,
+ }, nil
+ }
+
+ rows, err := queryScopeDuplicateRows(basicRes.GetDal(), connectionId,
githubIds)
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{
+ Body: ScopeDuplicatesOutput{Duplicates:
groupScopeDuplicateRows(rows)},
+ Status: http.StatusOK,
+ }, nil
+}
+
+func parseScopeDuplicateQuery(input *plugin.ApiResourceInput) (*uint64, []int,
errors.Error) {
+ var connectionId *uint64
+ if v := input.Query.Get("connectionId"); v != "" {
+ id, err := strconv.ParseUint(v, 10, 64)
+ if err != nil {
+ return nil, nil, errors.BadInput.Wrap(err, "invalid
connectionId")
+ }
+ connectionId = &id
+ }
+
+ var githubIds []int
+ if v := input.Query.Get("githubIds"); v != "" {
+ for _, part := range strings.Split(v, ",") {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ id, err := strconv.Atoi(part)
+ if err != nil {
+ return nil, nil, errors.BadInput.Wrap(err,
"invalid githubIds")
+ }
+ githubIds = append(githubIds, id)
+ }
+ }
+
+ if len(githubIds) > 0 && connectionId == nil {
+ return nil, nil, errors.BadInput.New("connectionId is required
when githubIds is provided")
+ }
+
+ return connectionId, githubIds, nil
+}
+
+// queryScopeDuplicateRows loads only the rows needed for the requested mode.
+// Check mode: selected githubIds on any connection other than connectionId.
+// Diagnostics: githubIds that already appear on more than one connection.
+func queryScopeDuplicateRows(db dal.Dal, connectionId *uint64, githubIds
[]int) ([]scopeDuplicateRow, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("r.github_id, r.html_url, r.full_name,
r.connection_id, c.name AS connection_name"),
+ dal.From("_tool_github_repos r"),
+ dal.Join("INNER JOIN _tool_github_connections c ON c.id =
r.connection_id"),
+ dal.Orderby("r.github_id ASC, r.connection_id ASC"),
+ }
+
+ if connectionId != nil {
+ clauses = append(clauses, dal.Where(
+ "r.github_id IN ? AND r.connection_id != ?",
+ githubIds,
+ *connectionId,
+ ))
+ } else {
+ clauses = append(clauses, dal.Where(`r.github_id IN (
+ SELECT github_id FROM _tool_github_repos
+ GROUP BY github_id
+ HAVING COUNT(DISTINCT connection_id) > 1
+ )`))
+ }
+
+ var rows []scopeDuplicateRow
+ if err := db.All(&rows, clauses...); err != nil {
+ return nil, err
+ }
+ return rows, nil
+}
+
+// groupScopeDuplicateRows collapses already-filtered SQL rows into API groups.
+func groupScopeDuplicateRows(rows []scopeDuplicateRow) []ScopeDuplicateGroup {
+ if len(rows) == 0 {
+ return []ScopeDuplicateGroup{}
+ }
+
+ result := make([]ScopeDuplicateGroup, 0)
+ var current *ScopeDuplicateGroup
+ seenConns := make(map[uint64]struct{})
+
+ flush := func() {
+ if current != nil {
+ result = append(result, *current)
+ }
+ }
+
+ for _, row := range rows {
+ if current == nil || current.GithubId != row.GithubId {
+ flush()
+ current = &ScopeDuplicateGroup{
+ GithubId: row.GithubId,
+ HTMLUrl: row.HTMLUrl,
+ FullName: row.FullName,
+ Connections: make([]ScopeDuplicateConnection,
0, 2),
+ }
+ seenConns = make(map[uint64]struct{})
+ }
+ if current.HTMLUrl == "" && row.HTMLUrl != "" {
+ current.HTMLUrl = row.HTMLUrl
+ }
+ if current.FullName == "" && row.FullName != "" {
+ current.FullName = row.FullName
+ }
+ if _, ok := seenConns[row.ConnectionId]; ok {
+ continue
+ }
+ seenConns[row.ConnectionId] = struct{}{}
+ current.Connections = append(current.Connections,
ScopeDuplicateConnection{
+ ConnectionId: row.ConnectionId,
+ ConnectionName: row.ConnectionName,
+ })
+ }
+ flush()
+ return result
+}
diff --git a/backend/plugins/github/api/scope_duplicates_api_test.go
b/backend/plugins/github/api/scope_duplicates_api_test.go
new file mode 100644
index 000000000..87cc33456
--- /dev/null
+++ b/backend/plugins/github/api/scope_duplicates_api_test.go
@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/url"
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGroupScopeDuplicateRows_Empty(t *testing.T) {
+ assert.Empty(t, groupScopeDuplicateRows(nil))
+ assert.Empty(t, groupScopeDuplicateRows([]scopeDuplicateRow{}))
+}
+
+func TestGroupScopeDuplicateRows_GroupsConnections(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GithubId: 100, HTMLUrl: "https://github.com/o/a";, FullName:
"o/a", ConnectionId: 1, ConnectionName: "GitHub Production"},
+ {GithubId: 100, HTMLUrl: "https://github.com/o/a";, FullName:
"o/a", ConnectionId: 2, ConnectionName: "GitHub Staging"},
+ {GithubId: 200, HTMLUrl: "https://github.com/o/b";, FullName:
"o/b", ConnectionId: 3, ConnectionName: "Other"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Equal(t, []ScopeDuplicateGroup{
+ {
+ GithubId: 100,
+ HTMLUrl: "https://github.com/o/a";,
+ FullName: "o/a",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "GitHub
Production"},
+ {ConnectionId: 2, ConnectionName: "GitHub
Staging"},
+ },
+ },
+ {
+ GithubId: 200,
+ HTMLUrl: "https://github.com/o/b";,
+ FullName: "o/b",
+ Connections: []ScopeDuplicateConnection{
+ {ConnectionId: 3, ConnectionName: "Other"},
+ },
+ },
+ }, got)
+}
+
+func TestGroupScopeDuplicateRows_DedupesSameConnection(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GithubId: 100, FullName: "o/a", ConnectionId: 1,
ConnectionName: "Prod"},
+ {GithubId: 100, FullName: "o/a", ConnectionId: 1,
ConnectionName: "Prod"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, []ScopeDuplicateConnection{
+ {ConnectionId: 1, ConnectionName: "Prod"},
+ }, got[0].Connections)
+}
+
+func TestGroupScopeDuplicateRows_FillsMissingLabels(t *testing.T) {
+ rows := []scopeDuplicateRow{
+ {GithubId: 100, ConnectionId: 1, ConnectionName: "A"},
+ {GithubId: 100, HTMLUrl: "https://github.com/o/a";, FullName:
"o/a", ConnectionId: 2, ConnectionName: "B"},
+ }
+
+ got := groupScopeDuplicateRows(rows)
+ assert.Len(t, got, 1)
+ assert.Equal(t, "https://github.com/o/a";, got[0].HTMLUrl)
+ assert.Equal(t, "o/a", got[0].FullName)
+}
+
+func TestParseScopeDuplicateQuery(t *testing.T) {
+ input := &plugin.ApiResourceInput{Query: url.Values{}}
+ connId, ids, err := parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Nil(t, connId)
+ assert.Empty(t, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"3"},
+ "githubIds": []string{"10, 20,30"},
+ }}
+ connId, ids, err = parseScopeDuplicateQuery(input)
+ assert.Nil(t, err)
+ assert.Equal(t, uint64(3), *connId)
+ assert.Equal(t, []int{10, 20, 30}, ids)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "githubIds": []string{"10"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Error(t, err)
+
+ input = &plugin.ApiResourceInput{Query: url.Values{
+ "connectionId": []string{"abc"},
+ }}
+ _, _, err = parseScopeDuplicateQuery(input)
+ assert.Error(t, err)
+}
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index e09779a3d..ada24da0d 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -225,6 +225,9 @@ func (p Github) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"scope-config/:scopeConfigId/projects": {
"GET": api.GetProjectsByScopeConfig,
},
+ "scope-duplicates": {
+ "GET": api.GetScopeDuplicates,
+ },
}
}
diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts
index 7007314f6..8996f4662 100644
--- a/config-ui/src/api/scope/index.ts
+++ b/config-ui/src/api/scope/index.ts
@@ -96,3 +96,27 @@ export const searchRemote = (
method: 'get',
data,
});
+
+export type ScopeDuplicateConnection = {
+ connectionId: ID;
+ connectionName: string;
+};
+
+export type ScopeDuplicateGroup = {
+ githubId: number;
+ htmlUrl: string;
+ fullName: string;
+ connections: ScopeDuplicateConnection[];
+};
+
+export const scopeDuplicates = (
+ plugin: string,
+ data?: {
+ connectionId?: ID;
+ githubIds?: string;
+ },
+): Promise<{ duplicates: ScopeDuplicateGroup[] }> =>
+ request(`/plugins/${plugin}/scope-duplicates`, {
+ method: 'get',
+ data,
+ });
diff --git
a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
index 6bf150916..7453cbedc 100644
--- a/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
+++ b/config-ui/src/plugins/components/data-scope-remote/data-scope-remote.tsx
@@ -17,9 +17,10 @@
*/
import { useState, useEffect, useMemo } from 'react';
-import { Flex, Button } from 'antd';
+import { Flex, Button, Alert } from 'antd';
import API from '@/api';
+import type { ScopeDuplicateGroup } from '@/api/scope';
import { getPluginConfig } from '@/plugins';
import { operator } from '@/utils';
@@ -38,6 +39,39 @@ interface Props {
onSubmit?: (origin: any) => void;
}
+const getGithubId = (scope: any): number | undefined => {
+ const fromData = scope?.data?.githubId;
+ if (typeof fromData === 'number' && fromData > 0) {
+ return fromData;
+ }
+ const fromId = Number(scope?.id);
+ if (!Number.isNaN(fromId) && fromId > 0) {
+ return fromId;
+ }
+ return undefined;
+};
+
+const buildDuplicateWarning = (duplicates: ScopeDuplicateGroup[]): string => {
+ const connectionNames = Array.from(
+ new Set(
+ duplicates.flatMap((d) => d.connections.map((c) =>
c.connectionName).filter(Boolean)),
+ ),
+ );
+ const repoLabel =
+ duplicates.length === 1
+ ? duplicates[0].fullName || duplicates[0].htmlUrl || 'This repository'
+ : 'One or more selected repositories';
+
+ const via =
+ connectionNames.length === 1
+ ? `Connection "${connectionNames[0]}"`
+ : connectionNames.length > 1
+ ? `Connections ${connectionNames.map((n) => `"${n}"`).join(', ')}`
+ : 'another connection';
+
+ return `${repoLabel} is already connected via ${via}. Collecting it here
will create duplicate pull requests and issue records, which will inflate all
metrics for this repository.`;
+};
+
export const DataScopeRemote = ({
mode = 'multiple',
plugin,
@@ -51,6 +85,8 @@ export const DataScopeRemote = ({
}: Props) => {
const [selectedScope, setSelectedScope] = useState<any[]>([]);
const [operating, setOperating] = useState(false);
+ const [duplicates, setDuplicates] = useState<ScopeDuplicateGroup[]>([]);
+ const [warningDismissed, setWarningDismissed] = useState(false);
useEffect(() => {
setSelectedScope(props.selectedScope ?? []);
@@ -58,6 +94,48 @@ export const DataScopeRemote = ({
const config = useMemo(() => getPluginConfig(plugin).dataScope, [plugin]);
+ const githubIdsKey = useMemo(() => {
+ if (plugin !== 'github') {
+ return '';
+ }
+ return selectedScope
+ .map(getGithubId)
+ .filter((id): id is number => id !== undefined)
+ .sort((a, b) => a - b)
+ .join(',');
+ }, [plugin, selectedScope]);
+
+ useEffect(() => {
+ if (plugin !== 'github' || !githubIdsKey) {
+ setDuplicates([]);
+ setWarningDismissed(false);
+ return;
+ }
+
+ let cancelled = false;
+ setWarningDismissed(false);
+
+ API.scope
+ .scopeDuplicates(plugin, {
+ connectionId,
+ githubIds: githubIdsKey,
+ })
+ .then((res) => {
+ if (!cancelled) {
+ setDuplicates(res.duplicates ?? []);
+ }
+ })
+ .catch(() => {
+ if (!cancelled) {
+ setDuplicates([]);
+ }
+ });
+
+ return () => {
+ cancelled = true;
+ };
+ }, [plugin, connectionId, githubIdsKey]);
+
const handleSubmit = async () => {
const [success, res] = await operator(
() => API.scope.batch(plugin, connectionId, { data:
selectedScope.map((it) => it.data) }),
@@ -72,8 +150,20 @@ export const DataScopeRemote = ({
}
};
+ const showWarning = plugin === 'github' && duplicates.length > 0 &&
!warningDismissed;
+
return (
<Flex vertical>
+ {showWarning && (
+ <Alert
+ type="warning"
+ showIcon
+ closable
+ onClose={() => setWarningDismissed(true)}
+ style={{ marginBottom: 16 }}
+ message={buildDuplicateWarning(duplicates)}
+ />
+ )}
{config.render ? (
config.render({
connectionId,
