This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-7852-components-field-length in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ea7a6899f8a9847d9c5d63d0d8892814460429ce Author: Klesh Wong <[email protected]> AuthorDate: Wed Aug 7 17:15:16 2024 +0800 refactor: update github job extractor for single record structure --- .../plugins/github_graphql/tasks/job_collector.go | 17 +++--- .../plugins/github_graphql/tasks/job_extractor.go | 67 ++++++++++------------ 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index 10694e56a..73c914a8b 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -160,15 +160,16 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { ResponseParser: func(queryWrapper any) (messages []json.RawMessage, err errors.Error) { query := queryWrapper.(*GraphqlQueryCheckRunWrapper) for _, node := range query.Node { - checkRun := node.CheckSuite.CheckRuns.Nodes[0] - updatedAt := checkRun.StartedAt - if checkRun.CompletedAt != nil { - updatedAt = checkRun.CompletedAt - } - if apiCollector.GetSince() != nil && !apiCollector.GetSince().Before(*updatedAt) { - return messages, helper.ErrFinishCollect + for _, checkRun := range node.CheckSuite.CheckRuns.Nodes { + updatedAt := checkRun.StartedAt + if checkRun.CompletedAt != nil { + updatedAt = checkRun.CompletedAt + } + if apiCollector.GetSince() != nil && !apiCollector.GetSince().Before(*updatedAt) { + return messages, helper.ErrFinishCollect + } + messages = append(messages, errors.Must1(json.Marshal(node))) } - messages = append(messages, errors.Must1(json.Marshal(node))) } return }, diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go b/backend/plugins/github_graphql/tasks/job_extractor.go index 4e38cc727..1bd035a7e 100644 --- a/backend/plugins/github_graphql/tasks/job_extractor.go +++ b/backend/plugins/github_graphql/tasks/job_extractor.go @@ -51,47 +51,42 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_GRAPHQL_JOBS_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - apiJob := &GraphqlQueryCheckRunWrapper{} - err := errors.Convert(json.Unmarshal(row.Data, apiJob)) + checkSuite := &GraphqlQueryCheckSuite{} + err := errors.Convert(json.Unmarshal(row.Data, checkSuite)) if err != nil { return nil, err } - - nodes := apiJob.Node results := make([]interface{}, 0, 1) - for _, node := range nodes { - for _, checkRun := range node.CheckSuite.CheckRuns.Nodes { - - paramsBytes, err := json.Marshal(checkRun.Steps.Nodes) - if err != nil { - taskCtx.GetLogger().Error(err, `Marshal checkRun.Steps.Nodes fail and ignore`) - } - githubJob := &models.GithubJob{ - ConnectionId: data.Options.ConnectionId, - RunID: node.CheckSuite.WorkflowRun.DatabaseId, - RepoId: data.Options.GithubId, - ID: checkRun.DatabaseId, - NodeID: checkRun.Id, - HTMLURL: checkRun.DetailsUrl, - Status: strings.ToUpper(checkRun.Status), - Conclusion: strings.ToUpper(checkRun.Conclusion), - StartedAt: checkRun.StartedAt, - CompletedAt: checkRun.CompletedAt, - Name: checkRun.Name, - Steps: paramsBytes, - Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, checkRun.Name), - Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, checkRun.Name), - // these columns can not fill by graphql - //HeadSha: ``, // use _tool_github_runs - //RunURL: ``, - //CheckRunURL: ``, - //Labels: ``, // not in use - //RunnerID: ``, // not in use - //RunnerName: ``, // not in use - //RunnerGroupID: ``, // not in use - } - results = append(results, githubJob) + for _, checkRun := range checkSuite.CheckSuite.CheckRuns.Nodes { + paramsBytes, err := json.Marshal(checkRun.Steps.Nodes) + if err != nil { + taskCtx.GetLogger().Error(err, `Marshal checkRun.Steps.Nodes fail and ignore`) + } + githubJob := &models.GithubJob{ + ConnectionId: data.Options.ConnectionId, + RunID: checkSuite.CheckSuite.WorkflowRun.DatabaseId, + RepoId: data.Options.GithubId, + ID: checkRun.DatabaseId, + NodeID: checkRun.Id, + HTMLURL: checkRun.DetailsUrl, + Status: strings.ToUpper(checkRun.Status), + Conclusion: strings.ToUpper(checkRun.Conclusion), + StartedAt: checkRun.StartedAt, + CompletedAt: checkRun.CompletedAt, + Name: checkRun.Name, + Steps: paramsBytes, + Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, checkRun.Name), + Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, checkRun.Name), + // these columns can not fill by graphql + //HeadSha: ``, // use _tool_github_runs + //RunURL: ``, + //CheckRunURL: ``, + //Labels: ``, // not in use + //RunnerID: ``, // not in use + //RunnerName: ``, // not in use + //RunnerGroupID: ``, // not in use } + results = append(results, githubJob) } return results, nil
