This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0d753c446 Fix/8538 GitHub zero time (#8583)
0d753c446 is described below
commit 0d753c446d1e7ee4f2aa0b241099cb28f9e87e5d
Author: Kostas Petrakis <[email protected]>
AuthorDate: Wed Sep 24 08:08:55 2025 +0200
Fix/8538 GitHub zero time (#8583)
* fix(github): fix zerotime issues for GraphQL
* fix(lint): fix linting issue
---------
Co-authored-by: Kostas Petrakis <[email protected]>
---
.../utils/time.go} | 38 +++++-----------
backend/core/utils/time_test.go | 51 ++++++++++++++++++++++
backend/plugins/github/models/release.go | 38 ++++++++--------
backend/plugins/github/tasks/release_convertor.go | 2 +-
.../github_graphql/tasks/account_collector.go | 15 +++----
.../tasks/account_graphql_pre_extractor.go | 1 -
.../github_graphql/tasks/issue_collector.go | 2 +
.../github_graphql/tasks/issue_extractor.go | 3 ++
.../plugins/github_graphql/tasks/job_collector.go | 14 +++---
.../plugins/github_graphql/tasks/job_extractor.go | 5 ++-
.../github_graphql/tasks/release_collector.go | 4 +-
.../github_graphql/tasks/release_extractor.go | 3 +-
12 files changed, 108 insertions(+), 68 deletions(-)
diff --git
a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
b/backend/core/utils/time.go
similarity index 52%
copy from backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
copy to backend/core/utils/time.go
index 92e4611cb..f1b95f499 100644
--- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
+++ b/backend/core/utils/time.go
@@ -15,34 +15,18 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package utils
-import (
- "github.com/apache/incubator-devlake/plugins/github/models"
-)
+import "time"
-type GithubAccountEdge struct {
- Login string
- Id int `graphql:"databaseId"`
- Name string
- Company string
- Email string
- AvatarUrl string
- HtmlUrl string `graphql:"url"`
- //Type string
-}
-type GraphqlInlineAccountQuery struct {
- GithubAccountEdge `graphql:"... on User"`
-}
-
-func extractGraphqlPreAccount(result *[]interface{}, res
*GraphqlInlineAccountQuery, repoId int, connId uint64) {
- if res == nil || res.Id == 0 {
- return
+// NilIfZeroTime returns nil if t is nil or represents the zero time
(0001-01-01...).
+// Otherwise, it returns t unchanged.
+func NilIfZeroTime(t *time.Time) *time.Time {
+ if t == nil {
+ return nil
+ }
+ if t.IsZero() {
+ return nil
}
- *result = append(*result, &models.GithubRepoAccount{
- ConnectionId: connId,
- RepoGithubId: repoId,
- Login: res.Login,
- AccountId: res.Id,
- })
+ return t
}
diff --git a/backend/core/utils/time_test.go b/backend/core/utils/time_test.go
new file mode 100644
index 000000000..a0e949f4e
--- /dev/null
+++ b/backend/core/utils/time_test.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package utils
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNilIfZeroTime(t *testing.T) {
+ type args struct {
+ t *time.Time
+ }
+ tests := []struct {
+ name string
+ args args
+ want *time.Time
+ }{
+ {
+ name: "Empty date should be nil",
+ args: args{nil},
+ want: nil,
+ },
+ {
+ name: "Zero date should be nil",
+ args: args{&time.Time{}},
+ want: nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, NilIfZeroTime(tt.args.t),
"NilIfZeroTime(%v)", tt.args.t)
+ })
+ }
+}
diff --git a/backend/plugins/github/models/release.go
b/backend/plugins/github/models/release.go
index 1e4ae1688..5442f9fda 100644
--- a/backend/plugins/github/models/release.go
+++ b/backend/plugins/github/models/release.go
@@ -25,25 +25,25 @@ import (
type GithubRelease struct {
common.NoPKModel `json:"-" mapstructure:"-"`
- ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
- GithubId int `json:"github_id"`
- Id string `json:"id"
gorm:"type:varchar(255);primaryKey"`
- AuthorName string `json:"authorName"`
- AuthorID string `json:"authorId"`
- CreatedAt time.Time `json:"createdAt"`
- DatabaseID int `json:"databaseId"`
- Description string `json:"description"`
- DescriptionHTML string `json:"descriptionHTML"`
- IsDraft bool `json:"isDraft"`
- IsLatest bool `json:"isLatest"`
- IsPrerelease bool `json:"isPrerelease"`
- Name string `json:"name"`
- PublishedAt time.Time `json:"publishedAt"`
- ResourcePath string `json:"resourcePath"`
- TagName string `json:"tagName"`
- UpdatedAt time.Time `json:"updatedAt"`
- CommitSha string `json:"commit_sha"`
- URL string `json:"url"`
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ GithubId int `json:"github_id"`
+ Id string `json:"id"
gorm:"type:varchar(255);primaryKey"`
+ AuthorName string `json:"authorName"`
+ AuthorID string `json:"authorId"`
+ CreatedAt time.Time `json:"createdAt"`
+ DatabaseID int `json:"databaseId"`
+ Description string `json:"description"`
+ DescriptionHTML string `json:"descriptionHTML"`
+ IsDraft bool `json:"isDraft"`
+ IsLatest bool `json:"isLatest"`
+ IsPrerelease bool `json:"isPrerelease"`
+ Name string `json:"name"`
+ PublishedAt *time.Time `json:"publishedAt"`
+ ResourcePath string `json:"resourcePath"`
+ TagName string `json:"tagName"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ CommitSha string `json:"commit_sha"`
+ URL string `json:"url"`
}
func (GithubRelease) TableName() string {
diff --git a/backend/plugins/github/tasks/release_convertor.go
b/backend/plugins/github/tasks/release_convertor.go
index 1a1389093..d0b5b4bb7 100644
--- a/backend/plugins/github/tasks/release_convertor.go
+++ b/backend/plugins/github/tasks/release_convertor.go
@@ -73,7 +73,7 @@ func ConvertRelease(taskCtx plugin.SubTaskContext)
errors.Error {
DomainEntity: domainlayer.DomainEntity{
Id:
releaseIdGen.Generate(githubRelease.ConnectionId, githubRelease.Id),
},
- PublishedAt: githubRelease.PublishedAt,
+ PublishedAt: *githubRelease.PublishedAt,
CicdScopeId:
releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId),
Name: githubRelease.Name,
DisplayTitle: githubRelease.Name,
diff --git a/backend/plugins/github_graphql/tasks/account_collector.go
b/backend/plugins/github_graphql/tasks/account_collector.go
index 27d71b65f..34e4faac1 100644
--- a/backend/plugins/github_graphql/tasks/account_collector.go
+++ b/backend/plugins/github_graphql/tasks/account_collector.go
@@ -40,14 +40,13 @@ type GraphqlQueryAccountWrapper struct {
}
type GraphqlQueryAccount struct {
- Login string
- Id int `graphql:"databaseId"`
- Name string
- Company string
- Email string
- AvatarUrl string
- HtmlUrl string `graphql:"url"`
- //Type string
+ Login string
+ Id int `graphql:"databaseId"`
+ Name string
+ Company string
+ Email string
+ AvatarUrl string
+ HtmlUrl string `graphql:"url"`
Organizations struct {
Nodes []struct {
Email string
diff --git
a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
index 92e4611cb..e4b47e475 100644
--- a/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
+++ b/backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go
@@ -29,7 +29,6 @@ type GithubAccountEdge struct {
Email string
AvatarUrl string
HtmlUrl string `graphql:"url"`
- //Type string
}
type GraphqlInlineAccountQuery struct {
GithubAccountEdge `graphql:"... on User"`
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go
b/backend/plugins/github_graphql/tasks/issue_collector.go
index 0945be007..bc4d7a17b 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -134,6 +135,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
query := queryWrapper.(*GraphqlQueryIssueWrapper)
issues := query.Repository.IssueList.Issues
for _, rawL := range issues {
+ rawL.ClosedAt =
utils.NilIfZeroTime(rawL.ClosedAt)
if since != nil && since.After(rawL.UpdatedAt) {
return messages, api.ErrFinishCollect
}
diff --git a/backend/plugins/github_graphql/tasks/issue_extractor.go
b/backend/plugins/github_graphql/tasks/issue_extractor.go
index 7ebd40822..0ebd28d18 100644
--- a/backend/plugins/github_graphql/tasks/issue_extractor.go
+++ b/backend/plugins/github_graphql/tasks/issue_extractor.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -68,6 +69,8 @@ func ExtractIssues(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, err
}
+ // Normalize zero-date to nil for closedAt
+ issue.ClosedAt = utils.NilIfZeroTime(issue.ClosedAt)
results := make([]interface{}, 0, 1)
githubIssue, err := convertGithubIssue(milestoneMap,
issue, data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go
b/backend/plugins/github_graphql/tasks/job_collector.go
index b204d55ba..c67d6a879 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -19,13 +19,13 @@ package tasks
import (
"encoding/json"
- "fmt"
"reflect"
"time"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -202,14 +202,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext)
errors.Error {
RunId: runId,
GraphqlQueryCheckRun: &checkRun,
}
- // A checkRun without a startedAt time
is a run that was never started (skipped)
- // akwardly, GitHub assigns a
completedAt time to such runs, which is the time when the run was skipped
- // TODO: Decide if we want to skip
those runs or should we assign the startedAt time to the completedAt time
- if dbCheckRun.StartedAt == nil ||
dbCheckRun.StartedAt.IsZero() {
- debug :=
fmt.Sprintf("collector: checkRun.StartedAt is nil or zero: %s", dbCheckRun.Id)
-
taskCtx.GetLogger().Debug(debug, "Collector: CheckRun started at is nil or
zero")
- continue
- }
+ // A checkRun without a startedAt time
is a run that was never started (skipped), GitHub returns
+ // a ZeroTime (Due to the GO
implementation) for startedAt, so we need to check for that here.
+ dbCheckRun.StartedAt =
utils.NilIfZeroTime(dbCheckRun.StartedAt)
+ dbCheckRun.CompletedAt =
utils.NilIfZeroTime(dbCheckRun.CompletedAt)
updatedAt := dbCheckRun.StartedAt
if dbCheckRun.CompletedAt != nil {
updatedAt =
dbCheckRun.CompletedAt
diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go
b/backend/plugins/github_graphql/tasks/job_extractor.go
index ac4401e06..ea4b7c7db 100644
--- a/backend/plugins/github_graphql/tasks/job_extractor.go
+++ b/backend/plugins/github_graphql/tasks/job_extractor.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -57,7 +58,9 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error {
return nil, err
}
results := make([]interface{}, 0, 1)
-
+ // Normalize zero-date times to nil
+ checkRun.StartedAt =
utils.NilIfZeroTime(checkRun.StartedAt)
+ checkRun.CompletedAt =
utils.NilIfZeroTime(checkRun.CompletedAt)
paramsBytes, marshalError :=
json.Marshal(checkRun.Steps.Nodes)
err = errors.Convert(marshalError)
if err != nil {
diff --git a/backend/plugins/github_graphql/tasks/release_collector.go
b/backend/plugins/github_graphql/tasks/release_collector.go
index 0090bd60c..83581c4c1 100644
--- a/backend/plugins/github_graphql/tasks/release_collector.go
+++ b/backend/plugins/github_graphql/tasks/release_collector.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
@@ -66,7 +67,7 @@ type GraphqlQueryRelease struct {
IsLatest bool `graphql:"isLatest"`
IsPrerelease bool `graphql:"isPrerelease"`
Name string `graphql:"name"`
- PublishedAt time.Time `graphql:"publishedAt"`
+ PublishedAt *time.Time `graphql:"publishedAt"`
ResourcePath string `graphql:"resourcePath"`
TagName string `graphql:"tagName"`
TagCommit GraphqlQueryReleaseTagCommit `graphql:"tagCommit"`
@@ -125,6 +126,7 @@ func CollectRelease(taskCtx plugin.SubTaskContext)
errors.Error {
query := queryWrapper.(*GraphqlQueryReleaseWrapper)
releases := query.Repository.Releases.Releases
for _, rawL := range releases {
+ rawL.PublishedAt =
utils.NilIfZeroTime(rawL.PublishedAt)
if since != nil && since.After(rawL.UpdatedAt) {
return messages, helper.ErrFinishCollect
}
diff --git a/backend/plugins/github_graphql/tasks/release_extractor.go
b/backend/plugins/github_graphql/tasks/release_extractor.go
index 4b819fc23..6fd4f25ac 100644
--- a/backend/plugins/github_graphql/tasks/release_extractor.go
+++ b/backend/plugins/github_graphql/tasks/release_extractor.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
githubModels "github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -55,7 +56,7 @@ func ExtractReleases(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, err
}
-
+ release.PublishedAt =
utils.NilIfZeroTime(release.PublishedAt)
var results []interface{}
githubRelease, err := convertGitHubRelease(release,
data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
