This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch release-v0.17
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.17 by this push:
new 98600185a fix: Fix extraction and convertion of running builds (#5348)
98600185a is described below
commit 98600185a8debad69cdb4b82e33e6a4e0aed51bd
Author: Camille Teruel <[email protected]>
AuthorDate: Fri Jun 2 18:55:13 2023 +0200
fix: Fix extraction and convertion of running builds (#5348)
Co-authored-by: Camille Teruel <[email protected]>
---
.../plugins/azuredevops/azuredevops/models.py | 29 ++++++++++++++++------
.../azuredevops/azuredevops/streams/builds.py | 10 +++++++-
.../azuredevops/azuredevops/streams/jobs.py | 10 +++++++-
3 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index 227da24f0..e61f58966 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -127,18 +127,31 @@ class Job(ToolModel, table=True):
id: str = Field(primary_key=True)
build_id: str = Field(primary_key=True)
name: str
- startTime: datetime.datetime
- finishTime: datetime.datetime
+ startTime: Optional[datetime.datetime]
+ finishTime: Optional[datetime.datetime]
state: JobState
- result: JobResult
+ result: Optional[JobResult]
@classmethod
def migrate(self, session: Session):
dialect = session.bind.dialect.name
+ if dialect not in ['mysql', 'postgresql']:
+ raise Exception(f'Unsupported dialect {dialect}')
+ table = self.__tablename__
+
+ # Add build_id column and add it tp primary key
if dialect == 'mysql':
- session.execute(f'ALTER TABLE {self.__tablename__} DROP PRIMARY
KEY')
+ session.execute(f'ALTER TABLE {table} DROP PRIMARY KEY')
elif dialect == 'postgresql':
- session.execute(f'ALTER TABLE {self.__tablename__} DROP CONSTRAINT
{self.__tablename__}_pkey')
- else:
- raise Exception(f'Unsupported dialect {dialect}')
- session.execute(f'ALTER TABLE {self.__tablename__} ADD PRIMARY KEY
(id, build_id)')
+ session.execute(f'ALTER TABLE {table} DROP CONSTRAINT
{table}_pkey')
+ session.execute(f'ALTER TABLE {table} ADD PRIMARY KEY (id, build_id)')
+
+ # Make columns nullable
+ if dialect == 'mysql':
+ session.execute(f'ALTER TABLE {table} MODIFY COLUMN startTime
DATETIME NULL')
+ session.execute(f'ALTER TABLE {table} MODIFY COLUMN finishTime
DATETIME NULL')
+ session.execute(f'ALTER TABLE {table} MODIFY COLUMN result
VARCHAR(255) NULL')
+ elif dialect == 'postgresql':
+ session.execute(f'ALTER TABLE {table} ALTER COLUMN "startTime"
DROP NOT NULL')
+ session.execute(f'ALTER TABLE {table} ALTER COLUMN "finishTime"
DROP NOT NULL')
+ session.execute(f'ALTER TABLE {table} ALTER COLUMN "result" DROP
NOT NULL')
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
index 97687f80a..22f762405 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
@@ -36,6 +36,9 @@ class Builds(Stream):
yield raw_build, state
def convert(self, b: Build, ctx: Context):
+ if not b.start_time:
+ return
+
result = None
if b.result == Build.BuildResult.Canceled:
result = devops.CICDResult.ABORT
@@ -65,13 +68,18 @@ class Builds(Stream):
if ctx.transformation_rule and
ctx.transformation_rule.production_pattern.search(b.name):
environment = devops.CICDEnvironment.PRODUCTION
+ if b.finish_time:
+ duration_sec = abs(b.finish_time.second-b.start_time.second)
+ else:
+ duration_sec = 0
+
yield devops.CICDPipeline(
name=b.name,
status=status,
created_date=b.start_time,
finished_date=b.finish_time,
result=result,
- duration_sec=abs(b.finish_time.second-b.start_time.second),
+ duration_sec=duration_sec,
environment=environment,
type=type,
cicd_scope_id=ctx.scope.domain_id(),
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
index 4d0b3a194..d1934745e 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
@@ -43,6 +43,9 @@ class Jobs(Substream):
yield raw_job, state
def convert(self, j: Job, ctx: Context) -> Iterable[devops.CICDPipeline]:
+ if not j.startTime:
+ return
+
result = None
if j.result == Job.JobResult.Abandoned:
result = devops.CICDResult.ABORT
@@ -72,6 +75,11 @@ class Jobs(Substream):
if ctx.transformation_rule and
ctx.transformation_rule.production_pattern.search(j.name):
environment = devops.CICDEnvironment.PRODUCTION
+ if j.finishTime:
+ duration_sec = abs(j.finishTime.second-j.startTime.second)
+ else:
+ duration_sec = 0
+
yield devops.CICDTask(
id=j.id,
name=j.name,
@@ -81,7 +89,7 @@ class Jobs(Substream):
finished_date=j.finishTime,
result=result,
type=type,
- duration_sec=abs(j.finishTime.second-j.startTime.second),
+ duration_sec=duration_sec,
environment=environment,
cicd_scope_id=ctx.scope.domain_id()
)
