This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v1.0 by this push:
new b34d5258d Gitextractor auth cherry (#8182)
b34d5258d is described below
commit b34d5258dee9d6703df6d2915703323fdaa6c1b5
Author: Caio Queiroz <[email protected]>
AuthorDate: Tue Nov 5 22:44:35 2024 -0300
Gitextractor auth cherry (#8182)
* feat: generating new github access token to every gitextractor task
Signed-off-by: Caio Queiroz <[email protected]>
* feat: using DynamicGitUrl interface to implement the git url logic
* refactor: remove unused code
* fix: unit test
* fix: lint
---------
Signed-off-by: Caio Queiroz <[email protected]>
---
backend/plugins/gitextractor/impl/impl.go | 27 +++++++++++
backend/plugins/gitextractor/parser/taskdata.go | 2 +
backend/plugins/github/api/blueprint_v200.go | 12 +++--
backend/plugins/github/impl/impl.go | 56 ++++++++++++++++++++++-
backend/plugins/gitlab/api/blueprint_V200_test.go | 5 ++
backend/plugins/gitlab/api/blueprint_v200.go | 12 +++--
6 files changed, 103 insertions(+), 11 deletions(-)
diff --git a/backend/plugins/gitextractor/impl/impl.go
b/backend/plugins/gitextractor/impl/impl.go
index 468e51734..18d2054ba 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -18,6 +18,7 @@ limitations under the License.
package impl
import (
+ "fmt"
"net/url"
"github.com/apache/incubator-devlake/core/dal"
@@ -37,6 +38,10 @@ var _ interface {
type GitExtractor struct{}
+type DynamicGitUrl interface {
+ GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64,
repoUrl string) (string, errors.Error)
+}
+
func (p GitExtractor) GetTablesInfo() []dal.Tabler {
return []dal.Tabler{}
}
@@ -68,6 +73,24 @@ func (p GitExtractor) PrepareTaskData(taskCtx
plugin.TaskContext, options map[st
return nil, err
}
+ if op.PluginName != "" {
+ pluginInstance, err := plugin.GetPlugin(op.PluginName)
+ if err != nil {
+ return nil, errors.Default.Wrap(err,
fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName))
+ }
+
+ if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok {
+ gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx,
op.ConnectionId, op.Url)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "failed to
get Git URL")
+ }
+
+ op.Url = gitUrl
+ } else {
+ log.Printf("Plugin does not implement DynamicGitUrl
interface for plugin: %s", op.PluginName)
+ }
+ }
+
parsedURL, err := giturls.Parse(op.Url)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed to parse git url")
@@ -122,3 +145,7 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext)
errors.Error {
func (p GitExtractor) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/gitextractor"
}
+
+func (p GitExtractor) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/gitextractor/parser/taskdata.go
b/backend/plugins/gitextractor/parser/taskdata.go
index 1b22c9885..8dccf5ffe 100644
--- a/backend/plugins/gitextractor/parser/taskdata.go
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -45,4 +45,6 @@ type GitExtractorOptions struct {
SkipCommitStat *bool `json:"skipCommitStat"
mapstructure:"skipCommitStat" comment:"skip all commit stat including
added/deleted lines and commit files as well"`
SkipCommitFiles *bool `json:"skipCommitFiles"
mapstructure:"skipCommitFiles"`
NoShallowClone bool `json:"noShallowClone"
mapstructure:"noShallowClone"`
+ ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId,omitempty"`
+ PluginName string `json:"pluginName"
mapstructure:"pluginName,omitempty"`
}
diff --git a/backend/plugins/github/api/blueprint_v200.go
b/backend/plugins/github/api/blueprint_v200.go
index 909323918..708a941b6 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -128,11 +128,13 @@ func makeDataSourcePipelinePlanV200(
stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "name": githubRepo.FullName,
- "fullName": githubRepo.FullName,
- "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
- "proxy": connection.Proxy,
+ "url": cloneUrl.String(),
+ "name": githubRepo.FullName,
+ "fullName": githubRepo.FullName,
+ "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID,
githubRepo.GithubId),
+ "proxy": connection.Proxy,
+ "connectionId": githubRepo.ConnectionId,
+ "pluginName": "github",
},
})
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index f73f33120..6c25cf238 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -19,6 +19,7 @@ package impl
import (
"fmt"
+ "strings"
"github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
@@ -243,9 +244,41 @@ func (p Github) Close(taskCtx plugin.TaskContext)
errors.Error {
return nil
}
+func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId
uint64, repoUrl string) (string, errors.Error) {
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ p.Name(),
+ )
+
+ connection := &models.GithubConnection{}
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return "", errors.Default.Wrap(err, "unable to get github
connection by the given connection ID")
+ }
+
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(),
connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx)
+ if err != nil {
+ return "", err
+ }
+
+ err = connection.PrepareApiClient(apiClient)
+ if err != nil {
+ return "", err
+ }
+
+ newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token)
+ if err != nil {
+ return "", err
+ }
+
+ return newUrl, nil
+}
+
func EnrichOptions(taskCtx plugin.TaskContext,
op *tasks.GithubOptions,
- apiClient *helper.ApiClient) errors.Error {
+ apiClient *helper.ApiClient,
+) errors.Error {
var githubRepo models.GithubRepo
// validate the op and set name=owner/repo if this is from advanced
mode or bpV100
err := tasks.ValidateTaskOptions(op)
@@ -310,3 +343,24 @@ func convertApiRepoToScope(repo *tasks.GithubApiRepo,
connectionId uint64) *mode
scope.CloneUrl = repo.CloneUrl
return &scope
}
+
+func replaceAcessTokenInUrl(gitURL, newCredential string) (string,
errors.Error) {
+ atIndex := strings.Index(gitURL, "@")
+ if atIndex == -1 {
+ return "", errors.Default.New("Invalid Git URL")
+ }
+
+ protocolIndex := strings.Index(gitURL, "://")
+ if protocolIndex == -1 {
+ return "", errors.Default.New("Invalid Git URL")
+ }
+
+ // Extract the base URL (e.g., "https://git:";)
+ baseURL := gitURL[:protocolIndex+7]
+
+ repoURL := gitURL[atIndex+1:]
+
+ modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL)
+
+ return modifiedURL, nil
+}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go
b/backend/plugins/gitlab/api/blueprint_V200_test.go
index 36b628e5d..fa4302cbb 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -130,6 +130,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
Name: gitlabProjectName,
PathWithNamespace: pathWithNamespace,
HttpUrlToRepo: httpUrlToRepo,
+ Scope: common.Scope{
+ ConnectionId:
connectionID,
+ },
},
ScopeConfig: scopeConfig,
},
@@ -166,6 +169,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
"name": gitlabProjectName,
"fullName": pathWithNamespace,
"url":
"https://git:nddtf@this_is_cloneUrl";,
+ "connectionId": connectionID,
+ "pluginName": pluginName,
},
},
},
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go
b/backend/plugins/gitlab/api/blueprint_v200.go
index 458d57bb3..b891f72b3 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -135,11 +135,13 @@ func makePipelinePlanV200(
stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "name": gitlabProject.Name,
- "fullName":
gitlabProject.PathWithNamespace,
- "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
gitlabProject.GitlabId),
- "proxy": connection.Proxy,
+ "url": cloneUrl.String(),
+ "name": gitlabProject.Name,
+ "fullName":
gitlabProject.PathWithNamespace,
+ "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
gitlabProject.GitlabId),
+ "proxy": connection.Proxy,
+ "connectionId":
gitlabProject.ConnectionId,
+ "pluginName": "gitlab",
},
})
}
