This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch release-v0.21
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.21 by this push:
new dd59084dd fix(azuredevops): fix some bugs (#7108) (#7111)
dd59084dd is described below
commit dd59084ddadf01c79a4ab34a5664827f5a44b521
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Mar 5 16:47:02 2024 +0800
fix(azuredevops): fix some bugs (#7108) (#7111)
Co-authored-by: Lynwee <[email protected]>
---
backend/plugins/gitextractor/parser/clone.go | 6 +++++-
.../python/plugins/azuredevops/azuredevops/main.py | 2 +-
.../azuredevops/azuredevops/streams/builds.py | 8 ++++++++
.../plugins/azuredevops/azuredevops/streams/jobs.py | 6 ++++++
.../azuredevops/streams/pull_request_commits.py | 12 ++++++++++--
.../azuredevops/streams/pull_requests.py | 6 ++++++
backend/python/pydevlake/pydevlake/api.py | 7 +++++++
backend/python/pydevlake/pydevlake/subtasks.py | 21 +++++++++++++++------
8 files changed, 58 insertions(+), 10 deletions(-)
diff --git a/backend/plugins/gitextractor/parser/clone.go
b/backend/plugins/gitextractor/parser/clone.go
index ba04429ca..f921a4745 100644
--- a/backend/plugins/gitextractor/parser/clone.go
+++ b/backend/plugins/gitextractor/parser/clone.go
@@ -122,7 +122,11 @@ func (l *GitRepoCreator) CloneOverHTTP(ctx
plugin.SubTaskContext, repoId, url, u
done <- struct{}{}
if err != nil {
// Some sensitive information such as password will be
released in this err.
- err = fmt.Errorf("plain clone git error")
+ if err.Error() == "repository not found" {
+ // do nothing, it's a safe error message.
+ } else {
+ err = fmt.Errorf("plain clone git error")
+ }
l.logger.Error(err, "PlainCloneContext")
return nil, err
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index d31b53b3d..aab9b0061 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -120,7 +120,7 @@ class AzureDevOpsPlugin(Plugin):
hint = None
try:
if connection.organization is None:
- hint = "You may need to edit your token to set organization to
'All accessible organizations"
+ hint = "You may need to edit your token to set organization to
'All accessible organizations'"
res = api.my_profile()
else:
hint = "Organization name may be incorrect or your token may
not have access to the organization."
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
index ef4717759..af2fdf252 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
@@ -30,8 +30,16 @@ class Builds(Stream):
def collect(self, state, context) -> Iterable[tuple[object, dict]]:
repo: GitRepository = context.scope
api = AzureDevOpsAPI(context.connection)
+ provider = repo.provider or 'tfsgit'
response = api.builds(repo.org_id, repo.project_id, repo.id,
repo.provider or 'tfsgit')
for raw_build in response:
+ raw_build["x_request_url"] = response.get_url_with_query_string()
+ raw_build["x_request_input"] = {
+ "OrgId": repo.org_id,
+ "ProjectId": repo.project_id,
+ "RepoId": repo.id,
+ "Provider": provider,
+ }
yield raw_build, state
def convert(self, b: Build, ctx: Context):
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
index 9c625c5f3..7d2cd99fe 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
@@ -50,6 +50,12 @@ class Jobs(Substream):
for raw_job in response.json["records"]:
if raw_job["type"] == "Job":
raw_job["build_id"] = parent.domain_id()
+ raw_job["x_request_url"] = response.get_url_with_query_string()
+ raw_job["x_request_input"] = {
+ "OrgId": repo.org_id,
+ "ProjectId": repo.project_id,
+ "BuildId": parent.id,
+ }
yield raw_job, state
def convert(self, j: Job, ctx: Context) -> Iterable[devops.CICDPipeline]:
diff --git
a/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
b/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
index bd5ee0be8..43df25c28 100644
---
a/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
+++
b/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
@@ -15,11 +15,11 @@
from typing import Iterable
+import pydevlake.domain_layer.code as code
from azuredevops.api import AzureDevOpsAPI
from azuredevops.models import GitPullRequest, GitPullRequestCommit,
GitRepository
from azuredevops.streams.pull_requests import GitPullRequests
from pydevlake import Substream, DomainType
-import pydevlake.domain_layer.code as code
class GitPullRequestCommits(Substream):
@@ -34,9 +34,17 @@ class GitPullRequestCommits(Substream):
def collect(self, state, context, parent: GitPullRequest) ->
Iterable[tuple[object, dict]]:
repo: GitRepository = context.scope
azuredevops_api = AzureDevOpsAPI(context.connection)
- response = azuredevops_api.git_repo_pull_request_commits(repo.org_id,
repo.project_id, repo.id, parent.pull_request_id)
+ response = azuredevops_api.git_repo_pull_request_commits(repo.org_id,
repo.project_id, repo.id,
+
parent.pull_request_id)
for raw_commit in response:
raw_commit["pull_request_id"] = parent.domain_id()
+ raw_commit["x_request_url"] = response.get_url_with_query_string()
+ raw_commit["x_request_input"] = {
+ "OrgId": repo.org_id,
+ "ProjectId": repo.project_id,
+ "RepoId": repo.id,
+ "PullRequestId": parent.pull_request_id,
+ }
yield raw_commit, state
def convert(self, commit: GitPullRequestCommit, context) ->
Iterable[code.PullRequestCommit]:
diff --git
a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
index 3ae9d97c1..f58776d99 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
@@ -34,6 +34,12 @@ class GitPullRequests(Stream):
repo: GitRepository = context.scope
response = api.git_repo_pull_requests(repo.org_id, repo.project_id,
repo.id)
for raw_pr in response:
+ raw_pr["x_request_url"] = response.get_url_with_query_string()
+ raw_pr["x_request_input"] = {
+ "OrgId": repo.org_id,
+ "ProjectId": repo.project_id,
+ "RepoId": repo.id,
+ }
yield raw_pr, state
def convert(self, pr: GitPullRequest, ctx):
diff --git a/backend/python/pydevlake/pydevlake/api.py
b/backend/python/pydevlake/pydevlake/api.py
index 89678c2df..10547cb8a 100644
--- a/backend/python/pydevlake/pydevlake/api.py
+++ b/backend/python/pydevlake/pydevlake/api.py
@@ -17,6 +17,7 @@
from __future__ import annotations
from typing import Optional, Union
+from urllib.parse import urlencode
from http import HTTPStatus
import json
import time
@@ -71,6 +72,12 @@ class Response:
def __str__(self):
return f'{self.request}: {self.status}'
+ def get_url_with_query_string(self) -> str:
+ url = self.request.url
+ if self.request.query_args is not None:
+ url = f'{url}?{urlencode(self.request.query_args)}'
+ return url
+
# Sentinel value to abort processing of requests/responses in hooks
ABORT = object()
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py
b/backend/python/pydevlake/pydevlake/subtasks.py
index 52cadb89b..c1552f5b6 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -14,19 +14,18 @@
# limitations under the License.
-from abc import abstractmethod
import json
+from abc import abstractmethod
from datetime import datetime
from typing import Tuple, Dict, Iterable, Generator
-
import sqlalchemy.sql as sql
from sqlmodel import Session, select
-from pydevlake.model import RawModel, ToolModel, DomainModel, SubtaskRun,
raw_data_params
+from pydevlake import logger
from pydevlake.context import Context
from pydevlake.message import RemoteProgress
-from pydevlake import logger
+from pydevlake.model import RawModel, ToolModel, DomainModel, SubtaskRun,
raw_data_params
class Subtask:
@@ -74,7 +73,7 @@ class Subtask:
# Send final progress
if progress != last_progress:
yield RemoteProgress(
- increment=progress-last_progress,
+ increment=progress - last_progress,
current=progress
)
except Exception as e:
@@ -144,9 +143,18 @@ class Collector(Subtask):
def process(self, data: object, session: Session, ctx: Context):
raw_model_class = self.stream.raw_model(session)
+ url, input_info = "", ""
+ if "x_request_url" in data:
+ url = data["x_request_url"]
+ del data["x_request_url"]
+ if "x_request_input" in data:
+ input_info = data["x_request_input"]
+ del data["x_request_input"]
raw_model = raw_model_class(
params=self._params(ctx),
- data=json.dumps(data).encode('utf8')
+ data=json.dumps(data).encode('utf8'),
+ url=url,
+ input=json.dumps(input_info).encode('utf8'),
)
session.add(raw_model)
@@ -186,6 +194,7 @@ class Extractor(Subtask):
model = self.stream.tool_model
session.execute(sql.delete(model).where(model.raw_data_params ==
self._params(ctx)))
+
class Convertor(Subtask):
@property
def verb(self):
