This is an automated email from the ASF dual-hosted git repository. petkostas pushed a commit to branch fix/8538-github-zero-time in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit b58f6eceb56f4231017e15d54b35706fddd2fffd Author: Kostas Petrakis <[email protected]> AuthorDate: Sat Aug 23 13:30:04 2025 +0200 fix(github): fix zerotime issues for GraphQL --- .../utils/time.go} | 38 +++++----------- backend/core/utils/time_test.go | 51 ++++++++++++++++++++++ backend/plugins/github/models/release.go | 38 ++++++++-------- backend/plugins/github/tasks/release_convertor.go | 2 +- .../github_graphql/tasks/account_collector.go | 15 +++---- .../tasks/account_graphql_pre_extractor.go | 1 - .../github_graphql/tasks/issue_collector.go | 2 + .../github_graphql/tasks/issue_extractor.go | 3 ++ .../plugins/github_graphql/tasks/job_collector.go | 14 +++--- .../plugins/github_graphql/tasks/job_extractor.go | 5 ++- .../github_graphql/tasks/release_collector.go | 4 +- .../github_graphql/tasks/release_extractor.go | 3 +- 12 files changed, 108 insertions(+), 68 deletions(-) diff --git a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go b/backend/core/utils/time.go similarity index 52% copy from backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go copy to backend/core/utils/time.go index 92e4611cb..f1b95f499 100644 --- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go +++ b/backend/core/utils/time.go @@ -15,34 +15,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tasks +package utils -import ( - "github.com/apache/incubator-devlake/plugins/github/models" -) +import "time" -type GithubAccountEdge struct { - Login string - Id int `graphql:"databaseId"` - Name string - Company string - Email string - AvatarUrl string - HtmlUrl string `graphql:"url"` - //Type string -} -type GraphqlInlineAccountQuery struct { - GithubAccountEdge `graphql:"... on User"` -} - -func extractGraphqlPreAccount(result *[]interface{}, res *GraphqlInlineAccountQuery, repoId int, connId uint64) { - if res == nil || res.Id == 0 { - return +// NilIfZeroTime returns nil if t is nil or represents the zero time (0001-01-01...). +// Otherwise, it returns t unchanged. +func NilIfZeroTime(t *time.Time) *time.Time { + if t == nil { + return nil + } + if t.IsZero() { + return nil } - *result = append(*result, &models.GithubRepoAccount{ - ConnectionId: connId, - RepoGithubId: repoId, - Login: res.Login, - AccountId: res.Id, - }) + return t } diff --git a/backend/core/utils/time_test.go b/backend/core/utils/time_test.go new file mode 100644 index 000000000..a0e949f4e --- /dev/null +++ b/backend/core/utils/time_test.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package utils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNilIfZeroTime(t *testing.T) { + type args struct { + t *time.Time + } + tests := []struct { + name string + args args + want *time.Time + }{ + { + name: "Empty date should be nil", + args: args{nil}, + want: nil, + }, + { + name: "Zero date should be nil", + args: args{&time.Time{}}, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, NilIfZeroTime(tt.args.t), "NilIfZeroTime(%v)", tt.args.t) + }) + } +} diff --git a/backend/plugins/github/models/release.go b/backend/plugins/github/models/release.go index 1e4ae1688..5442f9fda 100644 --- a/backend/plugins/github/models/release.go +++ b/backend/plugins/github/models/release.go @@ -25,25 +25,25 @@ import ( type GithubRelease struct { common.NoPKModel `json:"-" mapstructure:"-"` - ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"` - GithubId int `json:"github_id"` - Id string `json:"id" gorm:"type:varchar(255);primaryKey"` - AuthorName string `json:"authorName"` - AuthorID string `json:"authorId"` - CreatedAt time.Time `json:"createdAt"` - DatabaseID int `json:"databaseId"` - Description string `json:"description"` - DescriptionHTML string `json:"descriptionHTML"` - IsDraft bool `json:"isDraft"` - IsLatest bool `json:"isLatest"` - IsPrerelease bool `json:"isPrerelease"` - Name string `json:"name"` - PublishedAt time.Time `json:"publishedAt"` - ResourcePath string `json:"resourcePath"` - TagName string `json:"tagName"` - UpdatedAt time.Time `json:"updatedAt"` - CommitSha string `json:"commit_sha"` - URL string `json:"url"` + ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"` + GithubId int `json:"github_id"` + Id string `json:"id" gorm:"type:varchar(255);primaryKey"` + AuthorName string `json:"authorName"` + AuthorID string `json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + DatabaseID int `json:"databaseId"` + Description string `json:"description"` + DescriptionHTML string `json:"descriptionHTML"` + IsDraft bool `json:"isDraft"` + IsLatest bool `json:"isLatest"` + IsPrerelease bool `json:"isPrerelease"` + Name string `json:"name"` + PublishedAt *time.Time `json:"publishedAt"` + ResourcePath string `json:"resourcePath"` + TagName string `json:"tagName"` + UpdatedAt time.Time `json:"updatedAt"` + CommitSha string `json:"commit_sha"` + URL string `json:"url"` } func (GithubRelease) TableName() string { diff --git a/backend/plugins/github/tasks/release_convertor.go b/backend/plugins/github/tasks/release_convertor.go index 1a1389093..d0b5b4bb7 100644 --- a/backend/plugins/github/tasks/release_convertor.go +++ b/backend/plugins/github/tasks/release_convertor.go @@ -73,7 +73,7 @@ func ConvertRelease(taskCtx plugin.SubTaskContext) errors.Error { DomainEntity: domainlayer.DomainEntity{ Id: releaseIdGen.Generate(githubRelease.ConnectionId, githubRelease.Id), }, - PublishedAt: githubRelease.PublishedAt, + PublishedAt: *githubRelease.PublishedAt, CicdScopeId: releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId), Name: githubRelease.Name, DisplayTitle: githubRelease.Name, diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go index 27d71b65f..34e4faac1 100644 --- a/backend/plugins/github_graphql/tasks/account_collector.go +++ b/backend/plugins/github_graphql/tasks/account_collector.go @@ -40,14 +40,13 @@ type GraphqlQueryAccountWrapper struct { } type GraphqlQueryAccount struct { - Login string - Id int `graphql:"databaseId"` - Name string - Company string - Email string - AvatarUrl string - HtmlUrl string `graphql:"url"` - //Type string + Login string + Id int `graphql:"databaseId"` + Name string + Company string + Email string + AvatarUrl string + HtmlUrl string `graphql:"url"` Organizations struct { Nodes []struct { Email string diff --git a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go index 92e4611cb..e4b47e475 100644 --- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go +++ b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go @@ -29,7 +29,6 @@ type GithubAccountEdge struct { Email string AvatarUrl string HtmlUrl string `graphql:"url"` - //Type string } type GraphqlInlineAccountQuery struct { GithubAccountEdge `graphql:"... on User"` diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go index 0945be007..bc4d7a17b 100644 --- a/backend/plugins/github_graphql/tasks/issue_collector.go +++ b/backend/plugins/github_graphql/tasks/issue_collector.go @@ -26,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" @@ -134,6 +135,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { query := queryWrapper.(*GraphqlQueryIssueWrapper) issues := query.Repository.IssueList.Issues for _, rawL := range issues { + rawL.ClosedAt = utils.NilIfZeroTime(rawL.ClosedAt) if since != nil && since.After(rawL.UpdatedAt) { return messages, api.ErrFinishCollect } diff --git a/backend/plugins/github_graphql/tasks/issue_extractor.go b/backend/plugins/github_graphql/tasks/issue_extractor.go index 7ebd40822..0ebd28d18 100644 --- a/backend/plugins/github_graphql/tasks/issue_extractor.go +++ b/backend/plugins/github_graphql/tasks/issue_extractor.go @@ -25,6 +25,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" @@ -68,6 +69,8 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } + // Normalize zero-date to nil for closedAt + issue.ClosedAt = utils.NilIfZeroTime(issue.ClosedAt) results := make([]interface{}, 0, 1) githubIssue, err := convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId, data.Options.GithubId) if err != nil { diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index b204d55ba..c67d6a879 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -19,13 +19,13 @@ package tasks import ( "encoding/json" - "fmt" "reflect" "time" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" @@ -202,14 +202,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { RunId: runId, GraphqlQueryCheckRun: &checkRun, } - // A checkRun without a startedAt time is a run that was never started (skipped) - // akwardly, GitHub assigns a completedAt time to such runs, which is the time when the run was skipped - // TODO: Decide if we want to skip those runs or should we assign the startedAt time to the completedAt time - if dbCheckRun.StartedAt == nil || dbCheckRun.StartedAt.IsZero() { - debug := fmt.Sprintf("collector: checkRun.StartedAt is nil or zero: %s", dbCheckRun.Id) - taskCtx.GetLogger().Debug(debug, "Collector: CheckRun started at is nil or zero") - continue - } + // A checkRun without a startedAt time is a run that was never started (skipped), GitHub returns + // a ZeroTime (Due to the GO implementation) for startedAt, so we need to check for that here. + dbCheckRun.StartedAt = utils.NilIfZeroTime(dbCheckRun.StartedAt) + dbCheckRun.CompletedAt = utils.NilIfZeroTime(dbCheckRun.CompletedAt) updatedAt := dbCheckRun.StartedAt if dbCheckRun.CompletedAt != nil { updatedAt = dbCheckRun.CompletedAt diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go b/backend/plugins/github_graphql/tasks/job_extractor.go index ac4401e06..ea4b7c7db 100644 --- a/backend/plugins/github_graphql/tasks/job_extractor.go +++ b/backend/plugins/github_graphql/tasks/job_extractor.go @@ -24,6 +24,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" @@ -57,7 +58,9 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error { return nil, err } results := make([]interface{}, 0, 1) - + // Normalize zero-date times to nil + checkRun.StartedAt = utils.NilIfZeroTime(checkRun.StartedAt) + checkRun.CompletedAt = utils.NilIfZeroTime(checkRun.CompletedAt) paramsBytes, marshalError := json.Marshal(checkRun.Steps.Nodes) err = errors.Convert(marshalError) if err != nil { diff --git a/backend/plugins/github_graphql/tasks/release_collector.go b/backend/plugins/github_graphql/tasks/release_collector.go index 0090bd60c..83581c4c1 100644 --- a/backend/plugins/github_graphql/tasks/release_collector.go +++ b/backend/plugins/github_graphql/tasks/release_collector.go @@ -24,6 +24,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" "github.com/merico-dev/graphql" @@ -66,7 +67,7 @@ type GraphqlQueryRelease struct { IsLatest bool `graphql:"isLatest"` IsPrerelease bool `graphql:"isPrerelease"` Name string `graphql:"name"` - PublishedAt time.Time `graphql:"publishedAt"` + PublishedAt *time.Time `graphql:"publishedAt"` ResourcePath string `graphql:"resourcePath"` TagName string `graphql:"tagName"` TagCommit GraphqlQueryReleaseTagCommit `graphql:"tagCommit"` @@ -125,6 +126,7 @@ func CollectRelease(taskCtx plugin.SubTaskContext) errors.Error { query := queryWrapper.(*GraphqlQueryReleaseWrapper) releases := query.Repository.Releases.Releases for _, rawL := range releases { + rawL.PublishedAt = utils.NilIfZeroTime(rawL.PublishedAt) if since != nil && since.After(rawL.UpdatedAt) { return messages, helper.ErrFinishCollect } diff --git a/backend/plugins/github_graphql/tasks/release_extractor.go b/backend/plugins/github_graphql/tasks/release_extractor.go index 4b819fc23..6fd4f25ac 100644 --- a/backend/plugins/github_graphql/tasks/release_extractor.go +++ b/backend/plugins/github_graphql/tasks/release_extractor.go @@ -23,6 +23,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" githubModels "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" @@ -55,7 +56,7 @@ func ExtractReleases(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } - + release.PublishedAt = utils.NilIfZeroTime(release.PublishedAt) var results []interface{} githubRelease, err := convertGitHubRelease(release, data.Options.ConnectionId, data.Options.GithubId) if err != nil {
