This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-6551-gitlab-scope in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 1b41027c41eb77cfb2c59cf4f22777776fab5d7b Author: Klesh Wong <[email protected]> AuthorDate: Mon Dec 4 11:32:31 2023 +0800 fix: gitlab remote scope api not working correctly Closes #6551 --- .../api/ds_remote_api_scope_list_helper.go | 7 ++- .../api/ds_remote_api_scope_search_helper.go | 11 +++-- backend/plugins/gitlab/api/remote_api.go | 52 +++++++++++++++------- backend/plugins/gitlab/impl/impl.go | 5 +-- backend/plugins/gitlab/models/project.go | 6 ++- 5 files changed, 55 insertions(+), 26 deletions(-) diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go index 60634c53d..076b2c6f8 100644 --- a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go +++ b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go @@ -75,7 +75,7 @@ func (rsl *DsRemoteApiScopeListHelper[C, S, P]) Get(input *plugin.ApiResourceInp } errors.Must(json.Unmarshal(decoded, pageInfo)) } - scopes, nextPage, err := rsl.listRemoteScopes(connection, apiClient, groupId, *pageInfo) + children, nextPage, err := rsl.listRemoteScopes(connection, apiClient, groupId, *pageInfo) if err != nil { return nil, err } @@ -85,9 +85,12 @@ func (rsl *DsRemoteApiScopeListHelper[C, S, P]) Get(input *plugin.ApiResourceInp nextPageJson := errors.Must1(json.Marshal(nextPage)) nextPageToken = base64.StdEncoding.EncodeToString(nextPageJson) } + if children == nil { + children = []models.DsRemoteApiScopeListEntry[S]{} + } return &plugin.ApiResourceOutput{ Body: map[string]interface{}{ - "children": scopes, + "children": children, "nextPageToken": nextPageToken, }, }, nil diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_scope_search_helper.go b/backend/helpers/pluginhelper/api/ds_remote_api_scope_search_helper.go index b5cc67367..cb870c7d8 100644 --- a/backend/helpers/pluginhelper/api/ds_remote_api_scope_search_helper.go +++ b/backend/helpers/pluginhelper/api/ds_remote_api_scope_search_helper.go @@ -62,17 +62,20 @@ func (rss *DsRemoteApiScopeSearchHelper[C, S]) Get(input *plugin.ApiResourceInpu if e := vld.Struct(params); e != nil { return nil, errors.BadInput.Wrap(e, "invalid params") } - scopes, err := rss.searchRemoteScopes(apiClient, params) + children, err := rss.searchRemoteScopes(apiClient, params) if err != nil { return nil, err } + if children == nil { + children = []models.DsRemoteApiScopeListEntry[S]{} + } // the config-ui is expecting the parent id to be null - for i := range scopes { - scopes[i].ParentId = nil + for i := range children { + children[i].ParentId = nil } return &plugin.ApiResourceOutput{ Body: map[string]interface{}{ - "children": scopes, + "children": children, "page": params.Page, "pageSize": params.PageSize, }, diff --git a/backend/plugins/gitlab/api/remote_api.go b/backend/plugins/gitlab/api/remote_api.go index d401939c5..fa68c68d3 100644 --- a/backend/plugins/gitlab/api/remote_api.go +++ b/backend/plugins/gitlab/api/remote_api.go @@ -30,6 +30,8 @@ import ( "github.com/apache/incubator-devlake/plugins/gitlab/models" ) +const USERS_PREFIX = "user:" + type GitlabRemotePagination struct { Page int `json:"page" mapstructure:"page"` PerPage int `json:"per_page" mapstructure:"per_page"` @@ -64,23 +66,27 @@ func listGitlabRemoteScopes( } // load all groups unless groupId is user's own account - if page.Step == "group" && !strings.HasPrefix(groupId, "users/") { + if page.Step == "group" && !strings.HasPrefix(groupId, USERS_PREFIX) { children, nextPage, err = listGitlabRemoteGroups(connection, apiClient, groupId, page) if err != nil { return } - // no more groups - if nextPage == nil { - nextPage = &GitlabRemotePagination{ - Page: 1, - PerPage: page.PerPage, - Step: "project", - } - } - } else { - // load all project under the group or user's own account - children, nextPage, err = listGitlabRemoteProjects(connection, apiClient, groupId, page) } + if groupId == "" || nextPage != nil { + return + } + // no more groups, start to load projects under the group + nextPage = &GitlabRemotePagination{ + Page: 1, + PerPage: page.PerPage, + Step: "project", + } + var moreChild []dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject] + moreChild, nextPage, err = listGitlabRemoteProjects(connection, apiClient, groupId, page) + if err != nil { + return + } + children = append(children, moreChild...) return } @@ -101,16 +107,18 @@ func listGitlabRemoteGroups( // make users own account as a group children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: fmt.Sprintf("users/%v", apiClient.GetData("UserId")), + Id: USERS_PREFIX + fmt.Sprintf("%v", apiClient.GetData("UserId")), Name: apiClient.GetData("UserName").(string), FullName: apiClient.GetData("UserName").(string), }) } + var parentId *string if groupId == "" { apiPath = "groups" query.Set("top_level_only", "true") } else { apiPath = fmt.Sprintf("groups/%s/subgroups", groupId) + parentId = &groupId } res, err = apiClient.Get(apiPath, query, nil) var resGroups []models.GroupResponse @@ -119,6 +127,7 @@ func listGitlabRemoteGroups( children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{ Type: api.RAS_ENTRY_TYPE_GROUP, Id: fmt.Sprintf("%v", group.Id), + ParentId: parentId, Name: group.Name, FullName: group.FullPath, }) @@ -142,12 +151,15 @@ func listGitlabRemoteProjects( query.Set("archived", "false") query.Set("min_access_level", "20") // - if strings.HasPrefix(groupId, "users/") { - apiPath = fmt.Sprintf("%s/projects", groupId) + if strings.HasPrefix(groupId, USERS_PREFIX) { + apiPath = fmt.Sprintf("users/%s/projects", strings.TrimPrefix(groupId, USERS_PREFIX)) } else { - apiPath = fmt.Sprintf("/groups/%s/projects", groupId) + apiPath = fmt.Sprintf("groups/%s/projects", groupId) } res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, err + } var resProjects []models.GitlabApiProject errors.Must(api.UnmarshalResponse(res, &resProjects)) for _, project := range resProjects { @@ -158,11 +170,19 @@ func listGitlabRemoteProjects( } func toProjectModel(project *models.GitlabApiProject) dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject] { + var parentId string + if project.Namespace.Kind == "user" { + parentId = USERS_PREFIX + fmt.Sprintf("%v", project.Owner.ID) + } else { + parentId = fmt.Sprintf("%v", project.Namespace.ID) + } return dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{ Type: api.RAS_ENTRY_TYPE_SCOPE, Id: fmt.Sprintf("%v", project.GitlabId), + ParentId: &parentId, Name: project.Name, FullName: project.PathWithNamespace, + Data: project.ConvertApiScope(), } } diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go index 9095db6cf..625f0a133 100644 --- a/backend/plugins/gitlab/impl/impl.go +++ b/backend/plugins/gitlab/impl/impl.go @@ -167,10 +167,9 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i return nil, err } logger.Debug(fmt.Sprintf("Current project: %d", project.GitlabId)) - i := project.ConvertApiScope() - scope = i.(*models.GitlabProject) + scope := project.ConvertApiScope() scope.ConnectionId = op.ConnectionId - err = taskCtx.GetDal().CreateIfNotExist(&scope) + err = taskCtx.GetDal().CreateIfNotExist(scope) if err != nil { return nil, err } diff --git a/backend/plugins/gitlab/models/project.go b/backend/plugins/gitlab/models/project.go index 54908af74..c82957f02 100644 --- a/backend/plugins/gitlab/models/project.go +++ b/backend/plugins/gitlab/models/project.go @@ -72,7 +72,7 @@ func (p GitlabProject) ScopeParams() interface{} { } // Convert the API response to our DB model instance -func (gitlabApiProject GitlabApiProject) ConvertApiScope() plugin.ToolLayerScope { +func (gitlabApiProject GitlabApiProject) ConvertApiScope() *GitlabProject { p := &GitlabProject{} p.GitlabId = gitlabApiProject.GitlabId p.Name = gitlabApiProject.Name @@ -124,6 +124,10 @@ type GitlabApiProject struct { FullPath string `json:"full_path"` ParentID any `json:"parent_id"` } `json:"namespace"` + Owner struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"owner"` } type Permissions struct {
