This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new bc6f948a fix: cicd github update runs to run on pipeline (#3018)
bc6f948a is described below
commit bc6f948a00ae8177e5ed4fae1a308e86e1752d5b
Author: abeizn <[email protected]>
AuthorDate: Thu Sep 8 16:31:30 2022 +0800
fix: cicd github update runs to run on pipeline (#3018)
fix: cicd github update runs to run on pipeline
---
plugins/github/impl/impl.go | 1 -
...gister.go => 20220908_delete_pipeline_table.go} | 35 +++++--
plugins/github/models/migrationscripts/register.go | 1 +
plugins/github/tasks/cicd_pipeline_convertor.go | 35 ++++---
plugins/github/tasks/cicd_run_enricher.go | 110 ---------------------
5 files changed, 45 insertions(+), 137 deletions(-)
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index 07529262..ee8384b0 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -112,7 +112,6 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractRunsMeta,
tasks.CollectJobsMeta,
tasks.ExtractJobsMeta,
- tasks.EnrichPipelinesMeta,
tasks.ConvertPipelinesMeta,
tasks.ConvertTasksMeta,
tasks.EnrichPullRequestIssuesMeta,
diff --git a/plugins/github/models/migrationscripts/register.go
b/plugins/github/models/migrationscripts/20220908_delete_pipeline_table.go
similarity index 54%
copy from plugins/github/models/migrationscripts/register.go
copy to plugins/github/models/migrationscripts/20220908_delete_pipeline_table.go
index 439d77d4..9b039675 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/20220908_delete_pipeline_table.go
@@ -18,15 +18,34 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/migration"
+ "context"
+
+ "github.com/apache/incubator-devlake/errors"
+ "gorm.io/gorm"
)
-// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(addInitTables),
- new(addGithubRunsTable),
- new(addGithubJobsTable),
- new(addGithubPipelineTable),
+type GithubPipeline20220908 struct{}
+
+func (GithubPipeline20220908) TableName() string {
+ return "_tool_github_pipelines"
+}
+
+type deleteGithubPipelineTable struct{}
+
+func (u *deleteGithubPipelineTable) Up(ctx context.Context, db *gorm.DB) error
{
+ // create table
+ err := db.Migrator().DropTable(GithubPipeline20220908{})
+ if err != nil {
+ return errors.Default.Wrap(err, "delete table
_tool_github_pipelines error")
}
+ return nil
+
+}
+
+func (*deleteGithubPipelineTable) Version() uint64 {
+ return 20220908000001
+}
+
+func (*deleteGithubPipelineTable) Name() string {
+ return "Github delete github_pipelines table"
}
diff --git a/plugins/github/models/migrationscripts/register.go
b/plugins/github/models/migrationscripts/register.go
index 439d77d4..1d25e283 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/register.go
@@ -28,5 +28,6 @@ func All() []migration.Script {
new(addGithubRunsTable),
new(addGithubJobsTable),
new(addGithubPipelineTable),
+ new(deleteGithubPipelineTable),
}
}
diff --git a/plugins/github/tasks/cicd_pipeline_convertor.go
b/plugins/github/tasks/cicd_pipeline_convertor.go
index 65dca697..a4176aa6 100644
--- a/plugins/github/tasks/cicd_pipeline_convertor.go
+++ b/plugins/github/tasks/cicd_pipeline_convertor.go
@@ -35,7 +35,7 @@ var ConvertPipelinesMeta = core.SubTaskMeta{
Name: "convertPipelines",
EntryPoint: ConvertPipelines,
EnabledByDefault: true,
- Description: "Convert tool layer table github_pipelines into
domain layer table cicd_pipeline",
+ Description: "Convert tool layer table github_runs into domain
layer table cicd_pipeline",
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
@@ -44,8 +44,9 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
- pipeline := &githubModels.GithubPipeline{}
+ pipeline := &githubModels.GithubRun{}
cursor, err := db.Cursor(
+ dal.Select("name, head_sha, head_branch, status, conclusion,
github_created_at, github_updated_at"),
dal.From(pipeline),
dal.Where("repo_id = ? and connection_id=?", repoId,
data.Options.ConnectionId),
)
@@ -65,22 +66,20 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
},
Table: RAW_RUN_TABLE,
},
- InputRowType: reflect.TypeOf(githubModels.GithubPipeline{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubRun{}),
Input: cursor,
Convert: func(inputRow interface{}) ([]interface{}, error) {
- line := inputRow.(*githubModels.GithubPipeline)
+ line := inputRow.(*githubModels.GithubRun)
domainPipeline := &devops.CICDPipeline{
- DomainEntity: domainlayer.DomainEntity{Id:
pipelineIdGen.Generate(data.Options.ConnectionId, repoId, line.Branch,
line.Commit)},
- Name:
didgen.NewDomainIdGenerator(&githubModels.GithubRepo{}).
- Generate(line.ConnectionId,
line.RepoId),
- Type: line.Type,
- DurationSec: uint64(line.Duration),
- CreatedDate: *line.StartedDate,
- FinishedDate: line.FinishedDate,
+ DomainEntity: domainlayer.DomainEntity{Id:
pipelineIdGen.Generate(data.Options.ConnectionId, repoId, line.HeadBranch,
line.HeadSha)},
+ Name: line.Name,
+ Type: "CI/CD",
+ CreatedDate: *line.GithubCreatedAt,
+ FinishedDate: line.GithubUpdatedAt,
}
- if line.Result == "success" {
+ if line.Conclusion == "success" {
domainPipeline.Result = devops.SUCCESS
- } else if line.Result == "failure" || line.Result ==
"startup_failure" {
+ } else if line.Conclusion == "failure" ||
line.Conclusion == "startup_failure" {
domainPipeline.Result = devops.FAILURE
} else {
domainPipeline.Result = devops.ABORT
@@ -90,14 +89,14 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
domainPipeline.Status = devops.IN_PROGRESS
} else {
domainPipeline.Status = devops.DONE
+ domainPipeline.DurationSec =
uint64(line.GithubUpdatedAt.Sub(*line.GithubCreatedAt).Seconds())
}
domainPipelineProject := &devops.CiCDPipelineRepo{
- DomainEntity: domainlayer.DomainEntity{Id:
pipelineIdGen.Generate(data.Options.ConnectionId, repoId, line.Branch,
line.Commit)},
- CommitSha: line.Commit,
- Branch: line.Branch,
- Repo:
didgen.NewDomainIdGenerator(&githubModels.GithubRepo{}).
- Generate(line.ConnectionId,
line.RepoId),
+ DomainEntity: domainlayer.DomainEntity{Id:
pipelineIdGen.Generate(data.Options.ConnectionId, repoId, line.HeadBranch,
line.HeadSha)},
+ CommitSha: line.HeadSha,
+ Branch: line.HeadBranch,
+ Repo:
didgen.NewDomainIdGenerator(&githubModels.GithubRepo{}).Generate(data.Options.ConnectionId,
repoId),
}
return []interface{}{
diff --git a/plugins/github/tasks/cicd_run_enricher.go
b/plugins/github/tasks/cicd_run_enricher.go
deleted file mode 100644
index 516dcf9c..00000000
--- a/plugins/github/tasks/cicd_run_enricher.go
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "encoding/json"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/core/dal"
- githubModels "github.com/apache/incubator-devlake/plugins/github/models"
-)
-
-var EnrichPipelinesMeta = core.SubTaskMeta{
- Name: "enrichPipelines",
- EntryPoint: EnrichPipelines,
- EnabledByDefault: true,
- Description: "Create tool layer table github_pipelines from
github_runs",
- DomainTypes: []string{core.DOMAIN_TYPE_CICD},
-}
-
-func EnrichPipelines(taskCtx core.SubTaskContext) (err error) {
- db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GithubTaskData)
- repoId := data.Repo.GithubId
-
- cursor, err := db.Cursor(
- dal.Select("head_sha, head_branch, status, conclusion,
github_created_at, github_updated_at, run_attempt, run_started_at,
_raw_data_id"),
- dal.From(&githubModels.GithubRun{}),
- dal.Orderby("head_sha, github_created_at"),
- )
- if err != nil {
- return err
- }
- defer cursor.Close()
- taskCtx.SetProgress(0, -1)
-
- apiParamsJson, err := json.Marshal(GithubApiParams{
- ConnectionId: data.Options.ConnectionId,
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
- })
- if err != nil {
- return err
- }
-
- for cursor.Next() {
- entity := &githubModels.GithubPipeline{}
- var item githubModels.GithubRun
- err = db.Fetch(cursor, &item)
- if err != nil {
- return err
- }
-
- if item.HeadSha != entity.Commit {
- entity.NoPKModel.RawDataId = item.NoPKModel.RawDataId
- entity.NoPKModel.RawDataTable = RAW_RUN_TABLE
- entity.NoPKModel.RawDataParams = string(apiParamsJson)
- entity.ConnectionId = data.Options.ConnectionId
- entity.RepoId = repoId
- entity.Commit = item.HeadSha
- entity.Branch = item.HeadBranch
- entity.StartedDate = item.GithubCreatedAt
- entity.FinishedDate = item.GithubUpdatedAt
- entity.Status = item.Status
- if entity.Status == "completed" {
- entity.Duration =
float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds())
- }
- entity.Result = item.Conclusion
- // TODO
- entity.Type = "CI/CD"
- } else {
- if item.GithubCreatedAt.Before(*entity.StartedDate) {
- entity.StartedDate = item.GithubCreatedAt
- }
- if item.GithubUpdatedAt.After(*entity.FinishedDate) &&
item.Status == "completed" {
- entity.FinishedDate = item.GithubCreatedAt
- entity.Duration =
float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds())
- }
- if item.Status != "completed" {
- entity.Status = item.Status
- }
- if item.Conclusion != "success" {
- entity.Result = item.Conclusion
- }
-
- }
- err = db.CreateOrUpdate(entity)
- if err != nil {
- return err
- }
- taskCtx.IncProgress(1)
- }
-
- return err
-
-}
