This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4973be6c5 fix(github_graphql): remove stale local issue rows when an
issue no longer resolves (#8820)
4973be6c5 is described below
commit 4973be6c5e6a95cbc70860565ea0fb3d234b3cd3
Author: lee jaeyoung <[email protected]>
AuthorDate: Fri Apr 10 01:46:57 2026 +0900
fix(github_graphql): remove stale local issue rows when an issue no longer
resolves (#8820)
* fix(github_graphql): remove stale local issue rows
* fix(github_graphql): prevent stale issue cleanup from crossing repository
boundaries
The previous missing-issue cleanup only keyed deletes by connection and
issue identifiers, which could reach too broadly when an issue was transferred
between repositories under the same DevLake connection. This change scopes
cleanup to the source repository by carrying RepoId through the refresh path
and requiring the source RawDataOrigin parameters before deleting stale tool
rows.
Constraint: Keep the fix inside the existing github_graphql collector path
without schema changes
Rejected: Continue deleting by connection_id + github_id only | could
remove destination-repository rows after transfers
Rejected: Disable stale cleanup for all missing issues | would leave
deleted-issue failures recurring
Confidence: medium
Scope-risk: moderate
Directive: Preserve source-scope guards if future cleanup paths touch
transferred issues again
Tested: go test ./plugins/github_graphql/tasks; go build
./helpers/pluginhelper/api ./plugins/github_graphql/tasks
Not-tested: End-to-end transfer scenario against a live GitHub repository
---
.../helpers/pluginhelper/api/graphql_collector.go | 18 +-
.../pluginhelper/api/graphql_collector_test.go | 31 ++++
.../github_graphql/tasks/issue_collector.go | 193 ++++++++++++++++++++-
.../github_graphql/tasks/issue_collector_test.go | 139 +++++++++++++++
4 files changed, 375 insertions(+), 6 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/graphql_collector.go
b/backend/helpers/pluginhelper/api/graphql_collector.go
index fdd779097..28a5b2b9d 100644
--- a/backend/helpers/pluginhelper/api/graphql_collector.go
+++ b/backend/helpers/pluginhelper/api/graphql_collector.go
@@ -276,14 +276,18 @@ func (collector *GraphqlCollector) fetchAsync(reqData
*GraphqlRequestData, handl
}
if len(dataErrors) > 0 {
if !collector.args.IgnoreQueryErrors {
+ hasNonIgnorableDataErrors := false
for _, dataError := range dataErrors {
- if strings.Contains(dataError.Error(), "Could
not resolve to an Issue") {
- logger.Warn(nil, "Issue may have been
transferred.")
- } else {
-
collector.checkError(errors.Default.Wrap(dataError, `graphql query got error`))
+ if isIgnorableGraphqlQueryError(dataError) {
+ logger.Warn(nil, "Issue may have been
transferred or deleted.")
+ continue
}
+ hasNonIgnorableDataErrors = true
+
collector.checkError(errors.Default.Wrap(dataError, `graphql query got error`))
+ }
+ if hasNonIgnorableDataErrors {
+ return
}
- return
}
// else: error will deal by ResponseParserWithDataErrors
}
@@ -344,4 +348,8 @@ func (collector *GraphqlCollector) HasError() bool {
return len(collector.workerErrors) > 0
}
+func isIgnorableGraphqlQueryError(err error) bool {
+ return err != nil && strings.Contains(err.Error(), "Could not resolve
to an Issue")
+}
+
var _ plugin.SubTask = (*GraphqlCollector)(nil)
diff --git a/backend/helpers/pluginhelper/api/graphql_collector_test.go
b/backend/helpers/pluginhelper/api/graphql_collector_test.go
new file mode 100644
index 000000000..fcab4dce0
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/graphql_collector_test.go
@@ -0,0 +1,31 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsIgnorableGraphqlQueryError(t *testing.T) {
+ assert.True(t, isIgnorableGraphqlQueryError(errors.New("Could not
resolve to an Issue with the number of 17.")))
+ assert.False(t, isIgnorableGraphqlQueryError(errors.New("some other
graphql error")))
+ assert.False(t, isIgnorableGraphqlQueryError(nil))
+}
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go
b/backend/plugins/github_graphql/tasks/issue_collector.go
index f4db1d5b6..a65293698 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -20,11 +20,14 @@ package tasks
import (
"encoding/json"
"reflect"
+ "sort"
"strings"
"time"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -49,7 +52,8 @@ type GraphqlQueryIssueWrapper struct {
}
type GraphqlQueryIssueDetailWrapper struct {
- RateLimit struct {
+ requestedIssues map[int]missingGithubIssueRef
+ RateLimit struct {
Cost int
}
Repository struct {
@@ -84,6 +88,22 @@ type GraphqlQueryIssue struct {
} `graphql:"labels(first: 100)"`
}
+type missingGithubIssueRef struct {
+ ConnectionId uint64
+ RepoId int
+ GithubId int
+ Number int
+ RawDataOrigin common.RawDataOrigin
+}
+
+type missingGithubIssueCleanupScope struct {
+ ConnectionId uint64
+ RepoId int
+ GithubId int
+ Number int
+ RawDataOrigin common.RawDataOrigin
+}
+
var CollectIssuesMeta = plugin.SubTaskMeta{
Name: "Collect Issues",
EntryPoint: CollectIssues,
@@ -175,12 +195,20 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
ownerName := strings.Split(data.Options.Name, "/")
inputIssues := reqData.Input.([]interface{})
outputIssues := []map[string]interface{}{}
+ query.requestedIssues =
make(map[int]missingGithubIssueRef, len(inputIssues))
for _, i := range inputIssues {
inputIssue := i.(*models.GithubIssue)
outputIssues = append(outputIssues,
map[string]interface{}{
`number`:
graphql.Int(inputIssue.Number),
})
issueUpdatedAt[inputIssue.Number] =
inputIssue.GithubUpdatedAt
+ query.requestedIssues[inputIssue.Number] =
missingGithubIssueRef{
+ ConnectionId: inputIssue.ConnectionId,
+ RepoId: inputIssue.RepoId,
+ GithubId: inputIssue.GithubId,
+ Number: inputIssue.Number,
+ RawDataOrigin: inputIssue.RawDataOrigin,
+ }
}
variables := map[string]interface{}{
"issue": outputIssues,
@@ -193,10 +221,17 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
query := queryWrapper.(*GraphqlQueryIssueDetailWrapper)
issues := query.Repository.Issues
for _, rawL := range issues {
+ if rawL.DatabaseId == 0 || rawL.Number == 0 {
+ continue
+ }
if
rawL.UpdatedAt.After(issueUpdatedAt[rawL.Number]) {
messages = append(messages,
errors.Must1(json.Marshal(rawL)))
}
}
+ missingIssues :=
findMissingGithubIssues(query.requestedIssues, issues)
+ if len(missingIssues) > 0 {
+ err = cleanupMissingGithubIssues(db,
taskCtx.GetLogger(), missingIssues)
+ }
return
},
})
@@ -206,3 +241,159 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
return apiCollector.Execute()
}
+
+func findMissingGithubIssues(requestedIssues map[int]missingGithubIssueRef,
resolvedIssues []GraphqlQueryIssue) []missingGithubIssueRef {
+ if len(requestedIssues) == 0 {
+ return nil
+ }
+
+ resolvedNumbers := make(map[int]struct{}, len(resolvedIssues))
+ for _, issue := range resolvedIssues {
+ if issue.DatabaseId == 0 || issue.Number == 0 {
+ continue
+ }
+ resolvedNumbers[issue.Number] = struct{}{}
+ }
+
+ missingIssues := make([]missingGithubIssueRef, 0)
+ for number, issue := range requestedIssues {
+ if _, ok := resolvedNumbers[number]; ok {
+ continue
+ }
+ missingIssues = append(missingIssues, issue)
+ }
+ sort.Slice(missingIssues, func(i, j int) bool {
+ return missingIssues[i].Number < missingIssues[j].Number
+ })
+ return missingIssues
+}
+
+func cleanupMissingGithubIssues(db dal.Dal, logger log.Logger, issues
[]missingGithubIssueRef) errors.Error {
+ var allErrors []error
+ for _, issue := range issues {
+ scope, ok := buildMissingGithubIssueCleanupScope(issue)
+ if !ok {
+ logger.Warn(nil, "GitHub issue #%d no longer resolves
from the source API, but source scope is incomplete so stale cleanup is
skipped", issue.Number)
+ continue
+ }
+ logger.Warn(nil, "GitHub issue #%d no longer resolves from the
source API, deleting stale local data for the current repository scope",
issue.Number)
+ err := cleanupMissingGithubIssue(db, scope)
+ if err != nil {
+ allErrors = append(allErrors, err)
+ }
+ }
+ return errors.Default.Combine(allErrors)
+}
+
+func buildMissingGithubIssueCleanupScope(issue missingGithubIssueRef)
(*missingGithubIssueCleanupScope, bool) {
+ if issue.ConnectionId == 0 || issue.RepoId == 0 || issue.GithubId == 0
|| issue.RawDataOrigin.RawDataTable == "" || issue.RawDataOrigin.RawDataParams
== "" {
+ return nil, false
+ }
+ return &missingGithubIssueCleanupScope{
+ ConnectionId: issue.ConnectionId,
+ RepoId: issue.RepoId,
+ GithubId: issue.GithubId,
+ Number: issue.Number,
+ RawDataOrigin: issue.RawDataOrigin,
+ }, true
+}
+
+func (scope *missingGithubIssueCleanupScope) issueScopedClauses() []dal.Clause
{
+ return []dal.Clause{
+ dal.Where(
+ "connection_id = ? AND issue_id = ? AND _raw_data_table
= ? AND _raw_data_params = ?",
+ scope.ConnectionId,
+ scope.GithubId,
+ scope.RawDataOrigin.RawDataTable,
+ scope.RawDataOrigin.RawDataParams,
+ ),
+ }
+}
+
+func (scope *missingGithubIssueCleanupScope) assigneeScopedClauses()
[]dal.Clause {
+ return []dal.Clause{
+ dal.Where(
+ "connection_id = ? AND repo_id = ? AND issue_id = ? AND
_raw_data_table = ? AND _raw_data_params = ?",
+ scope.ConnectionId,
+ scope.RepoId,
+ scope.GithubId,
+ scope.RawDataOrigin.RawDataTable,
+ scope.RawDataOrigin.RawDataParams,
+ ),
+ }
+}
+
+func (scope *missingGithubIssueCleanupScope) githubIssueScopedClauses()
[]dal.Clause {
+ return []dal.Clause{
+ dal.Where(
+ "connection_id = ? AND repo_id = ? AND github_id = ?
AND _raw_data_table = ? AND _raw_data_params = ?",
+ scope.ConnectionId,
+ scope.RepoId,
+ scope.GithubId,
+ scope.RawDataOrigin.RawDataTable,
+ scope.RawDataOrigin.RawDataParams,
+ ),
+ }
+}
+
+func (scope *missingGithubIssueCleanupScope) rawDataScopedClauses()
[]dal.Clause {
+ if scope.RawDataOrigin.RawDataId == 0 {
+ return nil
+ }
+ return []dal.Clause{dal.Where("id = ?", scope.RawDataOrigin.RawDataId)}
+}
+
+func cleanupMissingGithubIssue(db dal.Dal, scope
*missingGithubIssueCleanupScope) errors.Error {
+ deleteByIssueId := func(model any, table string) errors.Error {
+ clauses := append([]dal.Clause{dal.From(table)},
scope.issueScopedClauses()...)
+ err := db.Delete(model, clauses...)
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to delete stale
github issue data from "+table)
+ }
+ return nil
+ }
+
+ err := deleteByIssueId(&models.GithubIssueComment{},
models.GithubIssueComment{}.TableName())
+ if err != nil {
+ return err
+ }
+ err = deleteByIssueId(&models.GithubIssueEvent{},
models.GithubIssueEvent{}.TableName())
+ if err != nil {
+ return err
+ }
+ err = deleteByIssueId(&models.GithubIssueLabel{},
models.GithubIssueLabel{}.TableName())
+ if err != nil {
+ return err
+ }
+ err = db.Delete(
+ &models.GithubIssueAssignee{},
+
append([]dal.Clause{dal.From(models.GithubIssueAssignee{}.TableName())},
scope.assigneeScopedClauses()...)...,
+ )
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to delete stale github
issue assignees")
+ }
+ err = db.Delete(
+ &models.GithubPrIssue{},
+
append([]dal.Clause{dal.From(models.GithubPrIssue{}.TableName())},
scope.issueScopedClauses()...)...,
+ )
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to delete stale github
pull request issue links")
+ }
+ err = db.Delete(
+ &models.GithubIssue{},
+
append([]dal.Clause{dal.From(models.GithubIssue{}.TableName())},
scope.githubIssueScopedClauses()...)...,
+ )
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to delete stale github
issue")
+ }
+ if rawDataClauses := scope.rawDataScopedClauses(); len(rawDataClauses)
> 0 {
+ err = db.Delete(
+ &api.RawData{},
+
append([]dal.Clause{dal.From(scope.RawDataOrigin.RawDataTable)},
rawDataClauses...)...,
+ )
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to delete stale
raw github issue")
+ }
+ }
+ return nil
+}
diff --git a/backend/plugins/github_graphql/tasks/issue_collector_test.go
b/backend/plugins/github_graphql/tasks/issue_collector_test.go
new file mode 100644
index 000000000..6649b3e39
--- /dev/null
+++ b/backend/plugins/github_graphql/tasks/issue_collector_test.go
@@ -0,0 +1,139 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFindMissingGithubIssues(t *testing.T) {
+ requestedIssues := map[int]missingGithubIssueRef{
+ 17: {
+ ConnectionId: 1,
+ RepoId: 101,
+ GithubId: 1700,
+ Number: 17,
+ RawDataOrigin: common.RawDataOrigin{
+ RawDataTable: "_raw_github_graphql_issues",
+ RawDataParams:
"{\"connectionId\":1,\"name\":\"repo-a\"}",
+ RawDataId: 10,
+ },
+ },
+ 18: {
+ ConnectionId: 1,
+ RepoId: 101,
+ GithubId: 1800,
+ Number: 18,
+ },
+ }
+
+ resolvedIssues := []GraphqlQueryIssue{
+ {DatabaseId: 1800, Number: 18},
+ }
+
+ missingIssues := findMissingGithubIssues(requestedIssues,
resolvedIssues)
+
+ if assert.Len(t, missingIssues, 1) {
+ assert.Equal(t, 17, missingIssues[0].Number)
+ assert.Equal(t, 1700, missingIssues[0].GithubId)
+ assert.Equal(t, 101, missingIssues[0].RepoId)
+ assert.Equal(t, uint64(10),
missingIssues[0].RawDataOrigin.RawDataId)
+ }
+}
+
+func TestFindMissingGithubIssuesSkipsZeroValueResponses(t *testing.T) {
+ requestedIssues := map[int]missingGithubIssueRef{
+ 17: {Number: 17},
+ 18: {Number: 18},
+ }
+
+ resolvedIssues := []GraphqlQueryIssue{
+ {},
+ {DatabaseId: 1800, Number: 18},
+ }
+
+ missingIssues := findMissingGithubIssues(requestedIssues,
resolvedIssues)
+
+ if assert.Len(t, missingIssues, 1) {
+ assert.Equal(t, 17, missingIssues[0].Number)
+ }
+}
+
+func TestBuildMissingGithubIssueCleanupScopeRequiresSourceRawDataParams(t
*testing.T) {
+ scope, ok := buildMissingGithubIssueCleanupScope(missingGithubIssueRef{
+ ConnectionId: 1,
+ RepoId: 101,
+ GithubId: 1700,
+ Number: 17,
+ RawDataOrigin: common.RawDataOrigin{
+ RawDataTable: "_raw_github_graphql_issues",
+ },
+ })
+
+ assert.False(t, ok)
+ assert.Nil(t, scope)
+}
+
+func TestBuildMissingGithubIssueCleanupScopeBuildsRepoScopedClauses(t
*testing.T) {
+ scope, ok := buildMissingGithubIssueCleanupScope(missingGithubIssueRef{
+ ConnectionId: 1,
+ RepoId: 101,
+ GithubId: 1700,
+ Number: 17,
+ RawDataOrigin: common.RawDataOrigin{
+ RawDataTable: "_raw_github_graphql_issues",
+ RawDataParams:
"{\"connectionId\":1,\"name\":\"repo-a\"}",
+ RawDataId: 10,
+ },
+ })
+ if !assert.True(t, ok) {
+ return
+ }
+
+ issueClause := scope.issueScopedClauses()[0]
+ issueWhere, ok := issueClause.Data.(dal.DalClause)
+ if assert.True(t, ok) {
+ assert.Equal(t, "connection_id = ? AND issue_id = ? AND
_raw_data_table = ? AND _raw_data_params = ?", issueWhere.Expr)
+ assert.Equal(t, []interface{}{uint64(1), 1700,
"_raw_github_graphql_issues", "{\"connectionId\":1,\"name\":\"repo-a\"}"},
issueWhere.Params)
+ }
+
+ assigneeClause := scope.assigneeScopedClauses()[0]
+ assigneeWhere, ok := assigneeClause.Data.(dal.DalClause)
+ if assert.True(t, ok) {
+ assert.Equal(t, "connection_id = ? AND repo_id = ? AND issue_id
= ? AND _raw_data_table = ? AND _raw_data_params = ?", assigneeWhere.Expr)
+ assert.Equal(t, []interface{}{uint64(1), 101, 1700,
"_raw_github_graphql_issues", "{\"connectionId\":1,\"name\":\"repo-a\"}"},
assigneeWhere.Params)
+ }
+
+ githubIssueClause := scope.githubIssueScopedClauses()[0]
+ githubIssueWhere, ok := githubIssueClause.Data.(dal.DalClause)
+ if assert.True(t, ok) {
+ assert.Equal(t, "connection_id = ? AND repo_id = ? AND
github_id = ? AND _raw_data_table = ? AND _raw_data_params = ?",
githubIssueWhere.Expr)
+ assert.Equal(t, []interface{}{uint64(1), 101, 1700,
"_raw_github_graphql_issues", "{\"connectionId\":1,\"name\":\"repo-a\"}"},
githubIssueWhere.Params)
+ }
+
+ rawDataClause := scope.rawDataScopedClauses()[0]
+ rawDataWhere, ok := rawDataClause.Data.(dal.DalClause)
+ if assert.True(t, ok) {
+ assert.Equal(t, "id = ?", rawDataWhere.Expr)
+ assert.Equal(t, []interface{}{uint64(10)}, rawDataWhere.Params)
+ }
+}