This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2edda2047 feat: generate JQL with user's time zone (#4954)
2edda2047 is described below
commit 2edda2047120af1f4995d9139f25d867a1895025
Author: Liang Zhang <[email protected]>
AuthorDate: Sun Apr 23 15:49:51 2023 +0800
feat: generate JQL with user's time zone (#4954)
---
backend/core/runner/db.go | 10 +-
backend/core/runner/db_test.go | 68 ++++++++++++++
backend/plugins/jira/tasks/issue_collector.go | 101 +++++++++++++++++++--
backend/plugins/jira/tasks/issue_collector_test.go | 22 ++++-
4 files changed, 186 insertions(+), 15 deletions(-)
diff --git a/backend/core/runner/db.go b/backend/core/runner/db.go
index b5ef01bb0..bca6cc153 100644
--- a/backend/core/runner/db.go
+++ b/backend/core/runner/db.go
@@ -85,7 +85,7 @@ func NewGormDbEx(configReader config.ConfigReader, logger
log.Logger, sessionCon
var db *gorm.DB
switch strings.ToLower(u.Scheme) {
case "mysql":
- dbUrl = fmt.Sprintf("%s@tcp(%s)%s?%s", getUserString(u),
u.Host, u.Path, u.RawQuery)
+ dbUrl = fmt.Sprintf("%s@tcp(%s)%s?%s", getUserString(u),
u.Host, u.Path, addLocal(u.Query()))
db, err = gorm.Open(mysql.Open(dbUrl), dbConfig)
case "postgresql", "postgres", "pg":
db, err = gorm.Open(postgres.Open(dbUrl), dbConfig)
@@ -114,3 +114,11 @@ func getUserString(u *url.URL) string {
}
return userString
}
+
+// addLocal adds loc=Local to the query string if it's not already there
+func addLocal(query url.Values) string {
+ if query.Get("loc") == "" {
+ query.Set("loc", "Local")
+ }
+ return query.Encode()
+}
diff --git a/backend/core/runner/db_test.go b/backend/core/runner/db_test.go
new file mode 100644
index 000000000..550a8fc3e
--- /dev/null
+++ b/backend/core/runner/db_test.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runner
+
+import (
+ "net/url"
+ "testing"
+)
+
+func Test_addLocal(t *testing.T) {
+ values, _ := url.ParseQuery("charset=utf8mb4&parseTime=True")
+ type args struct {
+ query url.Values
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ }{
+ {
+ name: "test add local",
+ args: args{
+ query: values,
+ },
+ want: "charset=utf8mb4&loc=Local&parseTime=True",
+ },
+ {
+ name: "test add local",
+ args: args{
+ query: url.Values{
+ "local": []string{"abc"},
+ },
+ },
+ want: "loc=Local&local=abc",
+ },
+ {
+ name: "test add local",
+ args: args{
+ query: url.Values{
+ "loc": []string{"abc"},
+ },
+ },
+ want: "loc=abc",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := addLocal(tt.args.query); got != tt.want {
+ t.Errorf("addLocal() = %v, want %v", got,
tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/jira/tasks/issue_collector.go
b/backend/plugins/jira/tasks/issue_collector.go
index 64d2d77e2..19f9259c9 100644
--- a/backend/plugins/jira/tasks/issue_collector.go
+++ b/backend/plugins/jira/tasks/issue_collector.go
@@ -20,6 +20,8 @@ package tasks
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/plugins/jira/models"
"io"
"net/http"
"net/url"
@@ -44,7 +46,7 @@ var CollectIssuesMeta = plugin.SubTaskMeta{
func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
-
+ logger := taskCtx.GetLogger()
collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
/*
@@ -68,7 +70,13 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
// IMPORTANT: we have to keep paginated data in a consistence order to
avoid data-missing, if we sort issues by
// `updated`, issue will be jumping between pages if it got updated
during the collection process
incremental := collectorWithState.IsIncremental()
- jql := buildJQL(data.TimeAfter,
collectorWithState.LatestState.LatestSuccessStart, incremental)
+ loc, err := getTimeZone(taskCtx)
+ if err != nil {
+ logger.Info("failed to get timezone, err: %v", err)
+ } else {
+ logger.Info("got user's timezone: %v", loc.String())
+ }
+ jql := buildJQL(data.TimeAfter,
collectorWithState.LatestState.LatestSuccessStart, incremental, loc)
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
ApiClient: data.ApiClient,
@@ -137,22 +145,95 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
}
// buildJQL build jql based on timeAfter and incremental mode
-func buildJQL(timeAfter, latestSuccessStart *time.Time, isIncremental bool)
string {
+func buildJQL(timeAfter, latestSuccessStart *time.Time, isIncremental bool,
location *time.Location) string {
jql := "ORDER BY created ASC"
var moment time.Time
if timeAfter != nil {
moment = *timeAfter
}
// if isIncremental is true, we should not collect data before
latestSuccessStart
- if isIncremental {
- // subtract 24 hours to avoid missing data due to time zone
difference
- latest := latestSuccessStart.Add(-24 * time.Hour)
- if latest.After(moment) {
- moment = latest
- }
+ if isIncremental && latestSuccessStart.After(moment) {
+ moment = *latestSuccessStart
}
if !moment.IsZero() {
- jql = fmt.Sprintf("updated >= '%s' %s",
moment.In(time.UTC).Format("2006/01/02 15:04"), jql)
+ if location != nil {
+ moment = moment.In(location)
+ } else {
+ moment = moment.In(time.UTC).Add(-24 * time.Hour)
+ }
+ jql = fmt.Sprintf("updated >= '%s' %s",
moment.Format("2006/01/02 15:04"), jql)
}
return jql
}
+
+// getTimeZone get user's timezone from jira API
+func getTimeZone(taskCtx plugin.SubTaskContext) (*time.Location, errors.Error)
{
+ data := taskCtx.GetData().(*JiraTaskData)
+ connectionId := data.Options.ConnectionId
+ var conn models.JiraConnection
+ err := taskCtx.GetDal().First(&conn, dal.Where("id = ?", connectionId))
+ if err != nil {
+ return nil, err
+ }
+ var resp *http.Response
+ var path string
+ var query url.Values
+ if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ path = "api/2/user"
+ query = url.Values{"username": []string{conn.Username}}
+ } else {
+ path = "api/3/user"
+ var accountId string
+ accountId, err = getAccountId(data.ApiClient, conn.Username)
+ if err != nil {
+ return nil, err
+ }
+ query = url.Values{"accountId": []string{accountId}}
+ }
+ resp, err = data.ApiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ var timeZone struct {
+ TimeZone string `json:"timeZone"`
+ }
+ err = errors.Convert(json.NewDecoder(resp.Body).Decode(&timeZone))
+ if err != nil {
+ return nil, err
+ }
+ tz, err := errors.Convert01(time.LoadLocation(timeZone.TimeZone))
+ if err != nil {
+ return nil, err
+ }
+ if tz == nil {
+ return nil, errors.Default.New(fmt.Sprintf("invalid time zone:
%s", timeZone.TimeZone))
+ }
+ return tz, nil
+}
+
+func getAccountId(client *api.ApiAsyncClient, username string) (string,
errors.Error) {
+ resp, err := client.Get("api/3/user/picker", url.Values{"query":
[]string{username}}, nil)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ var accounts struct {
+ Users []struct {
+ AccountID string `json:"accountId"`
+ AccountType string `json:"accountType"`
+ HTML string `json:"html"`
+ DisplayName string `json:"displayName"`
+ } `json:"users"`
+ Total int `json:"total"`
+ Header string `json:"header"`
+ }
+ err = errors.Convert(json.NewDecoder(resp.Body).Decode(&accounts))
+ if err != nil {
+ return "", err
+ }
+ if len(accounts.Users) == 0 {
+ return "", errors.Default.New("no user found")
+ }
+ return accounts.Users[0].AccountID, nil
+}
diff --git a/backend/plugins/jira/tasks/issue_collector_test.go
b/backend/plugins/jira/tasks/issue_collector_test.go
index 3dba84ed1..c9891943d 100644
--- a/backend/plugins/jira/tasks/issue_collector_test.go
+++ b/backend/plugins/jira/tasks/issue_collector_test.go
@@ -27,10 +27,12 @@ func Test_buildJQL(t *testing.T) {
timeAfter := base
add48 := base.Add(48 * time.Hour)
minus48 := base.Add(-48 * time.Hour)
+ loc, _ := time.LoadLocation("Asia/Shanghai")
type args struct {
timeAfter *time.Time
latestSuccessStart *time.Time
isIncremental bool
+ location *time.Location
}
tests := []struct {
name string
@@ -51,8 +53,9 @@ func Test_buildJQL(t *testing.T) {
timeAfter: nil,
latestSuccessStart: &add48,
isIncremental: true,
+ location: loc,
},
- want: "updated >= '2021/02/04 04:05' ORDER BY created
ASC",
+ want: "updated >= '2021/02/05 12:05' ORDER BY created
ASC",
},
{
name: "test incremental",
@@ -60,8 +63,9 @@ func Test_buildJQL(t *testing.T) {
timeAfter: &base,
latestSuccessStart: nil,
isIncremental: false,
+ location: loc,
},
- want: "updated >= '2021/02/03 04:05' ORDER BY created
ASC",
+ want: "updated >= '2021/02/03 12:05' ORDER BY created
ASC",
},
{
name: "test incremental",
@@ -72,6 +76,16 @@ func Test_buildJQL(t *testing.T) {
},
want: "updated >= '2021/02/04 04:05' ORDER BY created
ASC",
},
+ {
+ name: "test incremental",
+ args: args{
+ timeAfter: &timeAfter,
+ latestSuccessStart: &add48,
+ isIncremental: true,
+ location: loc,
+ },
+ want: "updated >= '2021/02/05 12:05' ORDER BY created
ASC",
+ },
{
name: "test incremental",
args: args{
@@ -79,13 +93,13 @@ func Test_buildJQL(t *testing.T) {
latestSuccessStart: &minus48,
isIncremental: true,
},
- want: "updated >= '2021/02/03 04:05' ORDER BY created
ASC",
+ want: "updated >= '2021/02/02 04:05' ORDER BY created
ASC",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := buildJQL(tt.args.timeAfter,
tt.args.latestSuccessStart, tt.args.isIncremental); got != tt.want {
+ if got := buildJQL(tt.args.timeAfter,
tt.args.latestSuccessStart, tt.args.isIncremental, tt.args.location); got !=
tt.want {
t.Errorf("buildJQL() = %v, want %v", got,
tt.want)
}
})
