This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2849e4581 feat: filter out the GitLab repos that users only have the
Guest perm… (#5664)
2849e4581 is described below
commit 2849e4581b94a95694052418c57685544660cf32
Author: abeizn <[email protected]>
AuthorDate: Tue Jul 25 14:01:24 2023 +0800
feat: filter out the GitLab repos that users only have the Guest perm…
(#5664)
* feat: filter out the GitLab repos that users only have the Guest
permission and archived projects
* fix: go lint
---
backend/plugins/gitlab/api/remote.go | 33 ++++++++++++++---
backend/plugins/gitlab/models/account.go | 23 ++++++++++++
...230712_add_archived_to_tool_gitlab_projects.go} | 42 +++++++++++-----------
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/project.go | 39 ++++++++++----------
backend/plugins/gitlab/tasks/project_convertor.go | 1 +
backend/plugins/gitlab/tasks/project_extractor.go | 1 +
7 files changed, 97 insertions(+), 43 deletions(-)
diff --git a/backend/plugins/gitlab/api/remote.go
b/backend/plugins/gitlab/api/remote.go
index 545e8d26f..2fb8499c5 100644
--- a/backend/plugins/gitlab/api/remote.go
+++ b/backend/plugins/gitlab/api/remote.go
@@ -81,11 +81,16 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
}
query := initialQuery(queryData)
var res *http.Response
+ var resBody []models.GitlabApiProject
if gid == "" {
res, err =
apiClient.Get(fmt.Sprintf("users/%d/projects", apiClient.GetData("UserId")),
query, nil)
if err != nil {
return nil, err
}
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
} else {
query.Set("with_shared", "false")
if gid[:6] == "group:" {
@@ -95,11 +100,29 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
if err != nil {
return nil, err
}
- }
- var resBody []models.GitlabApiProject
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ filteredProjects :=
make([]models.GitlabApiProject, 0)
+ for _, project := range resBody {
+ membersURL :=
fmt.Sprintf("/projects/%d/members/%d", project.GitlabId,
apiClient.GetData("UserId"))
+ membersRes, err :=
apiClient.Get(membersURL, nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ var member models.GitlabMember
+ err = api.UnmarshalResponse(membersRes,
&member)
+ if err != nil {
+ return nil, err
+ }
+ // Filter out projects where the user
only has Guest permission and archived projects
+ if member.AccessLevel != 10 &&
!project.Archived {
+ filteredProjects =
append(filteredProjects, project)
+ }
+ }
+ resBody = filteredProjects
}
return resBody, err
})
diff --git a/backend/plugins/gitlab/models/account.go
b/backend/plugins/gitlab/models/account.go
index 03db2e8ac..f9f8f1e33 100644
--- a/backend/plugins/gitlab/models/account.go
+++ b/backend/plugins/gitlab/models/account.go
@@ -38,3 +38,26 @@ type GitlabAccount struct {
func (GitlabAccount) TableName() string {
return "_tool_gitlab_accounts"
}
+
+type GitlabUser struct {
+ ID int `json:"id"`
+ Username string `json:"username"`
+ Name string `json:"name"`
+ State string `json:"state"`
+ AvatarURL string `json:"avatar_url"`
+ WebURL string `json:"web_url"`
+}
+
+type GitlabMember struct {
+ AccessLevel int `json:"access_level"`
+ CreatedAt string `json:"created_at"`
+ CreatedBy GitlabUser `json:"created_by"`
+ ExpiresAt string `json:"expires_at"`
+ ID int `json:"id"`
+ Username string `json:"username"`
+ Name string `json:"name"`
+ State string `json:"state"`
+ AvatarURL string `json:"avatar_url"`
+ WebURL string `json:"web_url"`
+ MembershipState string `json:"membership_state"`
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/20230712_add_archived_to_tool_gitlab_projects.go
similarity index 56%
copy from backend/plugins/gitlab/models/migrationscripts/register.go
copy to
backend/plugins/gitlab/models/migrationscripts/20230712_add_archived_to_tool_gitlab_projects.go
index f12be1883..516753d75 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++
b/backend/plugins/gitlab/models/migrationscripts/20230712_add_archived_to_tool_gitlab_projects.go
@@ -18,26 +18,28 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- new(addTransformationRule20221125),
- new(addStdTypeToIssue221230),
- new(addIsDetailRequired20230210),
- new(addConnectionIdToTransformationRule),
- new(addGitlabCommitAuthorInfo),
- new(addTypeEnvToPipeline),
- new(renameTr2ScopeConfig),
- new(addGitlabIssueAssignee),
- new(addMrCommitSha),
- new(addRawParamTableForScope),
- }
+type gitlabProject20230711 struct {
+ Archived bool `json:"archived"`
+}
+
+func (gitlabProject20230711) TableName() string {
+ return "_tool_gitlab_projects"
+}
+
+type addProjectArchived struct{}
+
+func (script *addProjectArchived) Up(basicRes context.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(&gitlabProject20230711{})
+}
+
+func (*addProjectArchived) Version() uint64 {
+ return 20230712095900
+}
+
+func (*addProjectArchived) Name() string {
+ return "add archived to _tool_gitlab_projects"
}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index f12be1883..9287ac8ea 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -39,5 +39,6 @@ func All() []plugin.MigrationScript {
new(addGitlabIssueAssignee),
new(addMrCommitSha),
new(addRawParamTableForScope),
+ new(addProjectArchived),
}
}
diff --git a/backend/plugins/gitlab/models/project.go
b/backend/plugins/gitlab/models/project.go
index 9e00cbad6..06a9a6456 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -30,24 +30,25 @@ import (
var _ plugin.ToolLayerScope = (*GitlabProject)(nil)
type GitlabProject struct {
- ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" validate:"required" gorm:"primaryKey"`
- ScopeConfigId uint64 `json:"scopeConfigId,omitempty"
mapstructure:"scopeConfigId"`
- GitlabId int `json:"gitlabId" mapstructure:"gitlabId"
validate:"required" gorm:"primaryKey"`
- Name string `json:"name" mapstructure:"name"
gorm:"type:varchar(255)"`
- Description string `json:"description"
mapstructure:"description"`
- DefaultBranch string `json:"defaultBranch"
mapstructure:"defaultBranch" gorm:"type:varchar(255)"`
- PathWithNamespace string `json:"pathWithNamespace"
mapstructure:"pathWithNamespace" gorm:"type:varchar(255)"`
- WebUrl string `json:"webUrl" mapstructure:"webUrl"
gorm:"type:varchar(255)"`
- CreatorId int `json:"creatorId"
mapstructure:"creatorId"`
- Visibility string `json:"visibility"
mapstructure:"visibility" gorm:"type:varchar(255)"`
- OpenIssuesCount int `json:"openIssuesCount"
mapstructure:"openIssuesCount"`
- StarCount int `json:"starCount"
mapstructure:"StarCount"`
- ForkedFromProjectId int `json:"forkedFromProjectId"
mapstructure:"forkedFromProjectId"`
- ForkedFromProjectWebUrl string `json:"forkedFromProjectWebUrl"
mapstructure:"forkedFromProjectWebUrl" gorm:"type:varchar(255)"`
- HttpUrlToRepo string `json:"httpUrlToRepo"
gorm:"type:varchar(255)"`
-
- CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
- UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" validate:"required" gorm:"primaryKey"`
+ ScopeConfigId uint64 `json:"scopeConfigId,omitempty"
mapstructure:"scopeConfigId"`
+ GitlabId int `json:"gitlabId"
mapstructure:"gitlabId" validate:"required" gorm:"primaryKey"`
+ Name string `json:"name" mapstructure:"name"
gorm:"type:varchar(255)"`
+ Description string `json:"description"
mapstructure:"description"`
+ DefaultBranch string `json:"defaultBranch"
mapstructure:"defaultBranch" gorm:"type:varchar(255)"`
+ PathWithNamespace string `json:"pathWithNamespace"
mapstructure:"pathWithNamespace" gorm:"type:varchar(255)"`
+ WebUrl string `json:"webUrl" mapstructure:"webUrl"
gorm:"type:varchar(255)"`
+ CreatorId int `json:"creatorId"
mapstructure:"creatorId"`
+ Visibility string `json:"visibility"
mapstructure:"visibility" gorm:"type:varchar(255)"`
+ OpenIssuesCount int `json:"openIssuesCount"
mapstructure:"openIssuesCount"`
+ StarCount int `json:"starCount"
mapstructure:"StarCount"`
+ ForkedFromProjectId int `json:"forkedFromProjectId"
mapstructure:"forkedFromProjectId"`
+ ForkedFromProjectWebUrl string `json:"forkedFromProjectWebUrl"
mapstructure:"forkedFromProjectWebUrl" gorm:"type:varchar(255)"`
+ HttpUrlToRepo string `json:"httpUrlToRepo"
gorm:"type:varchar(255)"`
+ CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ Archived bool `json:"archived"
mapstructure:"archived"`
+
common.NoPKModel `json:"-" mapstructure:"-"`
}
@@ -84,6 +85,7 @@ func (gitlabApiProject GitlabApiProject) ConvertApiScope()
plugin.ToolLayerScope
p.Visibility = gitlabApiProject.Visibility
p.OpenIssuesCount = gitlabApiProject.OpenIssuesCount
p.StarCount = gitlabApiProject.StarCount
+ p.Archived = gitlabApiProject.Archived
p.CreatedDate = gitlabApiProject.CreatedAt.ToNullableTime()
p.UpdatedDate =
helper.Iso8601TimeToTime(gitlabApiProject.LastActivityAt)
if gitlabApiProject.ForkedFromProject != nil {
@@ -112,6 +114,7 @@ type GitlabApiProject struct {
CreatedAt helper.Iso8601Time `json:"created_at"`
LastActivityAt *helper.Iso8601Time `json:"last_activity_at"`
HttpUrlToRepo string `json:"http_url_to_repo"`
+ Archived bool `json:"archived"`
}
type GroupResponse struct {
diff --git a/backend/plugins/gitlab/tasks/project_convertor.go
b/backend/plugins/gitlab/tasks/project_convertor.go
index f4990f99f..417d08a0a 100644
--- a/backend/plugins/gitlab/tasks/project_convertor.go
+++ b/backend/plugins/gitlab/tasks/project_convertor.go
@@ -54,6 +54,7 @@ type GitlabApiProject struct {
CreatedAt helper.Iso8601Time `json:"created_at"`
LastActivityAt *helper.Iso8601Time `json:"last_activity_at"`
HttpUrlToRepo string `json:"http_url_to_repo"`
+ Archived bool `json:"archived"`
}
var ConvertProjectMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/gitlab/tasks/project_extractor.go
b/backend/plugins/gitlab/tasks/project_extractor.go
index d8fb39e56..d1f142349 100644
--- a/backend/plugins/gitlab/tasks/project_extractor.go
+++ b/backend/plugins/gitlab/tasks/project_extractor.go
@@ -38,6 +38,7 @@ func ConvertProject(gitlabApiProject *GitlabApiProject)
*models.GitlabProject {
StarCount: gitlabApiProject.StarCount,
CreatedDate: gitlabApiProject.CreatedAt.ToNullableTime(),
UpdatedDate:
helper.Iso8601TimeToTime(gitlabApiProject.LastActivityAt),
+ Archived: gitlabApiProject.Archived,
}
if gitlabApiProject.ForkedFromProject != nil {
gitlabProject.ForkedFromProjectId =
gitlabApiProject.ForkedFromProject.GitlabId
