This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 52278b0a2 fix(bitbucket): migrate off cross-workspace APIs removed by
CHANGE-2770 (#8916)
52278b0a2 is described below
commit 52278b0a2b0e4b83c8edc3e0ad50f95d57172ecf
Author: Kenneth Wong <[email protected]>
AuthorDate: Sat Jul 11 19:56:02 2026 +0800
fix(bitbucket): migrate off cross-workspace APIs removed by CHANGE-2770
(#8916)
* fix(bitbucket): migrate off cross-workspace APIs removed by CHANGE-2770
- listBitbucketWorkspaces: replace GET /user/permissions/workspaces with
the supported GET /workspaces (lists the current user's workspaces). Workspace
slug/name now parsed from the top level of each value instead of a nested
"workspace" object.
- searchBitbucketRepos: replace cross-workspace GET
/repositories?role=member with per-workspace GET /repositories/{workspace},
enumerating the user's workspaces first and aggregating up to PageSize matches.
- models: flatten GroupResponse to match the /workspaces response shape.
* fix(bitbucket): list workspaces via supported /user/workspaces
(CHANGE-2770)
GET /2.0/workspaces is also deprecated, not just
/user/permissions/workspaces.
The supported replacement for enumerating the current user's workspaces is
GET /2.0/user/workspaces, which nests slug/name under a workspace object.
- remote_api.go: listBitbucketWorkspaces / listAllBitbucketWorkspaces now
call
/user/workspaces with workspace.* fields and read via
GroupId()/GroupName().
- models/repo.go: re-nest GroupResponse under WorkspaceItem to match the
/user/workspaces response shape.
Repo listing stays on the supported GET /2.0/repositories/{workspace}.
* fix(bitbucket): drop sort/fields on /user/workspaces (400 Invalid field
name)
GET /2.0/user/workspaces rejects sort=workspace.slug with HTTP 400 'Invalid
field name'. The bare paginated call returns the nested workspace objects we
parse, so drop the sort/fields query params in both listBitbucketWorkspaces
and listAllBitbucketWorkspaces.
* fix(bitbucket): ignore 410 Gone on sunset issues API
Bitbucket has sunset the issue tracker/wiki API; the issues endpoint now
returns 410 Gone instead of 404. ignoreHTTPStatus404 only mapped 404 to
ErrIgnoreAndContinue, so a 410 fell through to nil, DoAsync retried 3x and
failed the whole pipeline ("Collect Issues ended unexpectedly").
Treat 410 like 404 (ignore and continue). Shared helper also covers issue
comments and pr commits. Adds a table test for the status-code mapping.
---
backend/plugins/bitbucket/api/remote_api.go | 117 +++++++++++++++------
backend/plugins/bitbucket/models/repo.go | 10 +-
backend/plugins/bitbucket/tasks/api_common.go | 5 +-
backend/plugins/bitbucket/tasks/api_common_test.go | 53 ++++++++++
4 files changed, 146 insertions(+), 39 deletions(-)
diff --git a/backend/plugins/bitbucket/api/remote_api.go
b/backend/plugins/bitbucket/api/remote_api.go
index 211901ec3..cf60b1bd5 100644
--- a/backend/plugins/bitbucket/api/remote_api.go
+++ b/backend/plugins/bitbucket/api/remote_api.go
@@ -67,11 +67,15 @@ func listBitbucketWorkspaces(
err errors.Error,
) {
var res *http.Response
+ // /user/permissions/workspaces was removed and /workspaces deprecated
by
+ // Bitbucket CHANGE-2770; /user/workspaces lists the current user's
workspaces
+ // and is the supported replacement.
res, err = apiClient.Get(
- "/user/permissions/workspaces",
+ "/user/workspaces",
url.Values{
- "sort": {"workspace.slug"},
- "fields":
{"values.workspace.slug,values.workspace.name,pagelen,page,size"},
+ // No sort/fields: /user/workspaces rejects
sort=workspace.slug with
+ // HTTP 400 "Invalid field name". The bare call returns
the nested
+ // workspace objects we need; WorkspaceResponse ignores
extra fields.
"page": {fmt.Sprintf("%v", page.Page)},
"pagelen": {fmt.Sprintf("%v", page.PageLen)},
},
@@ -98,9 +102,9 @@ func listBitbucketWorkspaces(
for _, r := range resBody.Values {
children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
Type: api.RAS_ENTRY_TYPE_GROUP,
- Id: r.Workspace.Slug,
- Name: r.Workspace.Name,
- FullName: r.Workspace.Name,
+ Id: r.GroupId(),
+ Name: r.GroupName(),
+ FullName: r.GroupName(),
})
}
return
@@ -152,6 +156,10 @@ func listBitbucketRepos(
return
}
+// searchBitbucketRepos searches repositories by name across the user's
workspaces.
+// The cross-workspace GET /repositories?role=member was removed by Bitbucket
+// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped
+// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize
hits.
func searchBitbucketRepos(
apiClient plugin.ApiClient,
params *dsmodels.DsRemoteApiScopeSearchParams,
@@ -159,39 +167,84 @@ func searchBitbucketRepos(
children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo],
err errors.Error,
) {
- var res *http.Response
- res, err = apiClient.Get(
- "/repositories",
- url.Values{
- "sort": {"name"},
- "fields":
{"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"},
- "role": {"member"},
- "q": {fmt.Sprintf(`full_name~"%s"`,
params.Search)},
- "page": {fmt.Sprintf("%v", params.Page)},
- "pagelen": {fmt.Sprintf("%v", params.PageSize)},
- },
- nil,
- )
- if err != nil {
- return nil, err
+ pageSize := params.PageSize
+ if pageSize == 0 {
+ pageSize = 100
}
- var resBody models.ReposResponse
- err = api.UnmarshalResponse(res, &resBody)
+
+ workspaces, err := listAllBitbucketWorkspaces(apiClient)
if err != nil {
- return
+ return nil, err
}
- for _, r := range resBody.Values {
- children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
- Type: api.RAS_ENTRY_TYPE_SCOPE,
- Id: r.FullName,
- Name: r.Name,
- FullName: r.FullName,
- Data: r.ConvertApiScope(),
- })
+
+ for _, workspace := range workspaces {
+ if len(children) >= pageSize {
+ break
+ }
+ var res *http.Response
+ res, err = apiClient.Get(
+ fmt.Sprintf("/repositories/%s", workspace),
+ url.Values{
+ "sort": {"name"},
+ "fields":
{"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"},
+ "q": {fmt.Sprintf(`name~"%s"`,
params.Search)},
+ "pagelen": {fmt.Sprintf("%v", pageSize)},
+ },
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ var resBody models.ReposResponse
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return
+ }
+ for _, r := range resBody.Values {
+ children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
+ Type: api.RAS_ENTRY_TYPE_SCOPE,
+ Id: r.FullName,
+ Name: r.Name,
+ FullName: r.FullName,
+ Data: r.ConvertApiScope(),
+ })
+ }
}
return
}
+// listAllBitbucketWorkspaces returns every workspace slug accessible to the
+// authenticated user, following pagination of GET /2.0/user/workspaces.
+func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string,
errors.Error) {
+ var slugs []string
+ for page := 1; ; page++ {
+ res, err := apiClient.Get(
+ "/user/workspaces",
+ url.Values{
+ // No sort/fields (see
listBitbucketWorkspaces): /user/workspaces
+ // returns 400 on sort=workspace.slug.
+ "page": {fmt.Sprintf("%v", page)},
+ "pagelen": {"100"},
+ },
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ var resBody models.WorkspaceResponse
+ if err = api.UnmarshalResponse(res, &resBody); err != nil {
+ return nil, err
+ }
+ for _, w := range resBody.Values {
+ slugs = append(slugs, w.GroupId())
+ }
+ if len(resBody.Values) == 0 || page*resBody.Pagelen >=
resBody.Size {
+ break
+ }
+ }
+ return slugs, nil
+}
+
// RemoteScopes list all available scopes on the remote server
// @Summary list all available scopes on the remote server
// @Description list all available scopes on the remote server
diff --git a/backend/plugins/bitbucket/models/repo.go
b/backend/plugins/bitbucket/models/repo.go
index 799b549b9..b33df3615 100644
--- a/backend/plugins/bitbucket/models/repo.go
+++ b/backend/plugins/bitbucket/models/repo.go
@@ -117,17 +117,15 @@ type WorkspaceResponse struct {
Values []GroupResponse `json:"values"`
}
+// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported
+// replacement after Bitbucket CHANGE-2770 removed the cross-workspace
+// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces.
+// Each value nests the workspace slug/name under a "workspace" object.
type GroupResponse struct {
- //Type string `json:"type"`
- //Permission string `json:"permission"`
- //LastAccessed time.Time `json:"last_accessed"`
- //AddedOn time.Time `json:"added_on"`
Workspace WorkspaceItem `json:"workspace"`
}
type WorkspaceItem struct {
- //Type string `json:"type"`
- //Uuid string `json:"uuid"`
Slug string `json:"slug" group:"id"`
Name string `json:"name" group:"name"`
}
diff --git a/backend/plugins/bitbucket/tasks/api_common.go
b/backend/plugins/bitbucket/tasks/api_common.go
index 352f41cb9..ad21c9b1e 100644
--- a/backend/plugins/bitbucket/tasks/api_common.go
+++ b/backend/plugins/bitbucket/tasks/api_common.go
@@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error {
if res.StatusCode == http.StatusUnauthorized {
return errors.Unauthorized.New("authentication failed, please
check your AccessToken")
}
- if res.StatusCode == http.StatusNotFound {
+ // 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the
issue
+ // tracker/wiki API, so the endpoint is permanently removed for the
repo.
+ // Both mean "nothing to collect" — skip gracefully instead of retrying.
+ if res.StatusCode == http.StatusNotFound || res.StatusCode ==
http.StatusGone {
return api.ErrIgnoreAndContinue
}
return nil
diff --git a/backend/plugins/bitbucket/tasks/api_common_test.go
b/backend/plugins/bitbucket/tasks/api_common_test.go
new file mode 100644
index 000000000..9d15e52e9
--- /dev/null
+++ b/backend/plugins/bitbucket/tasks/api_common_test.go
@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIgnoreHTTPStatus404(t *testing.T) {
+ cases := []struct {
+ name string
+ statusCode int
+ wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip,
no retry)
+ wantErr bool // expect a real error
+ }{
+ {"404 no issue tracker -> ignore", http.StatusNotFound, true,
false},
+ {"410 issue tracker sunset -> ignore", http.StatusGone, true,
false},
+ {"401 unauthorized -> error", http.StatusUnauthorized, false,
true},
+ {"200 ok -> continue", http.StatusOK, false, false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ err := ignoreHTTPStatus404(&http.Response{StatusCode:
tc.statusCode})
+ switch {
+ case tc.wantIgnore:
+ assert.Equal(t, api.ErrIgnoreAndContinue, err)
+ case tc.wantErr:
+ assert.Error(t, err)
+ default:
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
