This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a4ff6d4b fix: ignore Issue HttpStatus404 when message is “Repository
has no issue tracker.” (#3064)
a4ff6d4b is described below
commit a4ff6d4bc2153e9fe35df6c64d1cb15fdcd1c12e
Author: tsoc <[email protected]>
AuthorDate: Wed Sep 14 20:05:35 2022 +0800
fix: ignore Issue HttpStatus404 when message is “Repository has no issue
tracker.” (#3064)
* fix: ignoreIssueHttpStatus404
* fix: change error type
* fix: fix line 157 in bitbucket/api_common.go(fmt.Sprintf format %s reads
arg #2, but call has 1 arg (govet))
* refactor: deleted utils/utils.go and move BitbucketPagination from
api_client.go to api_common.go
* fix: ignore httpstatus404 directly
---
plugins/bitbucket/tasks/api_client.go | 8 --
plugins/bitbucket/tasks/api_common.go | 18 +++
plugins/bitbucket/tasks/issue_collector.go | 4 +-
plugins/bitbucket/tasks/issue_comment_collector.go | 1 +
plugins/bitbucket/utils/utils.go | 152 ---------------------
5 files changed, 21 insertions(+), 162 deletions(-)
diff --git a/plugins/bitbucket/tasks/api_client.go
b/plugins/bitbucket/tasks/api_client.go
index 99b8aeb2..3a6ec47c 100644
--- a/plugins/bitbucket/tasks/api_client.go
+++ b/plugins/bitbucket/tasks/api_client.go
@@ -60,11 +60,3 @@ func CreateApiClient(taskCtx core.TaskContext, connection
*models.BitbucketConne
}
return asyncApiClient, nil
}
-
-type BitbucketPagination struct {
- Values []interface{} `json:"values"`
- PageLen int `json:"pagelen"`
- Size int `json:"size"`
- Page int `json:"page"`
- Next string `json:"next"`
-}
diff --git a/plugins/bitbucket/tasks/api_common.go
b/plugins/bitbucket/tasks/api_common.go
index 7c01b2dc..b8b487e1 100644
--- a/plugins/bitbucket/tasks/api_common.go
+++ b/plugins/bitbucket/tasks/api_common.go
@@ -40,6 +40,14 @@ type BitbucketInput struct {
BitbucketId int
}
+type BitbucketPagination struct {
+ Values []interface{} `json:"values"`
+ PageLen int `json:"pagelen"`
+ Size int `json:"size"`
+ Page int `json:"page"`
+ Next string `json:"next"`
+}
+
func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string)
(*helper.RawDataSubTaskArgs, *BitbucketTaskData) {
data := taskCtx.GetData().(*BitbucketTaskData)
RawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
@@ -136,3 +144,13 @@ func GetIssuesIterator(taskCtx core.SubTaskContext)
(*helper.DalCursorIterator,
return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(BitbucketInput{}))
}
+
+func ignoreHTTPStatus404(res *http.Response) errors.Error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return errors.Unauthorized.New("authentication failed, please
check your AccessToken")
+ }
+ if res.StatusCode == http.StatusNotFound {
+ return helper.ErrIgnoreAndContinue
+ }
+ return nil
+}
diff --git a/plugins/bitbucket/tasks/issue_collector.go
b/plugins/bitbucket/tasks/issue_collector.go
index 94e3c0fb..762a1139 100644
--- a/plugins/bitbucket/tasks/issue_collector.go
+++ b/plugins/bitbucket/tasks/issue_collector.go
@@ -19,9 +19,8 @@ package tasks
import (
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
-
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
)
const RAW_ISSUE_TABLE = "bitbucket_api_issues"
@@ -46,6 +45,7 @@ func CollectApiIssues(taskCtx core.SubTaskContext)
errors.Error {
Query: GetQuery,
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: GetRawMessageFromResponse,
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
diff --git a/plugins/bitbucket/tasks/issue_comment_collector.go
b/plugins/bitbucket/tasks/issue_comment_collector.go
index 01a0698c..84c86309 100644
--- a/plugins/bitbucket/tasks/issue_comment_collector.go
+++ b/plugins/bitbucket/tasks/issue_comment_collector.go
@@ -53,6 +53,7 @@ func CollectApiIssueComments(taskCtx core.SubTaskContext)
errors.Error {
Query: GetQuery,
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: GetRawMessageFromResponse,
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
diff --git a/plugins/bitbucket/utils/utils.go b/plugins/bitbucket/utils/utils.go
deleted file mode 100644
index 223bd330..00000000
--- a/plugins/bitbucket/utils/utils.go
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
- "net/http"
- "regexp"
- "strconv"
- "strings"
- "time"
-)
-
-type PagingInfo struct {
- Next int
- Last int
- First int
- Prev int
-}
-
-type RateLimitInfo struct {
- Date time.Time
- ResetTime time.Time
- Remaining int
-}
-
-func GetTotalPagesFromResponse(res *http.Response, args
*helper.ApiCollectorArgs) (int, errors.Error) {
- link := res.Header.Get("link")
- pageInfo, err := GetPagingFromLinkHeader(link)
- if err != nil {
- return 0, nil
- }
- return pageInfo.Last, nil
-}
-
-func ConvertRateLimitInfo(date string, resetTime string, remaining string)
(RateLimitInfo, errors.Error) {
- var rateLimitInfo RateLimitInfo
- var err errors.Error
- if date != "" {
- rateLimitInfo.Date, err = errors.Convert01(http.ParseTime(date))
- if err != nil {
- return rateLimitInfo, err
- }
- } else {
- return rateLimitInfo, errors.Default.New("rate limit date was
an empty string")
- }
- if resetTime != "" {
- resetInt, err := errors.Convert01(strconv.ParseInt(resetTime,
10, 64))
- if err != nil {
- return rateLimitInfo, err
- }
- rateLimitInfo.ResetTime = time.Unix(resetInt, 0)
- } else {
- return rateLimitInfo, errors.Default.New("rate limit reset time
was an empty string")
- }
- if remaining != "" {
- rateLimitInfo.Remaining, err = ConvertStringToInt(remaining)
- if err != nil {
- return rateLimitInfo, err
- }
- } else {
- return rateLimitInfo, errors.Default.New("rate remaining was an
empty string")
- }
- return rateLimitInfo, nil
-}
-
-func GetRateLimitPerSecond(info RateLimitInfo) int {
- unixResetTime := info.ResetTime.Unix()
- unixNow := info.Date.Unix()
- timeBetweenNowAndReset := unixResetTime - unixNow
- // Adjust the remaining to be less then actual to avoid hitting the
limit exactly.
- multiplier := 0.98
- adjustedRemaining := float64(info.Remaining) * multiplier
- return int(adjustedRemaining / float64(timeBetweenNowAndReset)) //*
multiplier
-}
-func ConvertStringToInt(input string) (int, errors.Error) {
- return errors.Convert01(strconv.Atoi(input))
-}
-func GetPagingFromLinkHeader(link string) (PagingInfo, errors.Error) {
- result := PagingInfo{
- Next: 1,
- Last: 1,
- Prev: 1,
- First: 1,
- }
- linksArray := strings.Split(link, ",")
- pattern1 := regexp.MustCompile(`page=*[0-9]+`)
- pattern2 := regexp.MustCompile(`rel="*[a-z]+`)
- if len(linksArray) >= 2 {
- for i := 0; i < len(linksArray); i++ {
- content := []byte(linksArray[i])
- loc1 := pattern1.FindIndex(content)
- loc2 := pattern2.FindIndex(content)
- if len(loc1) >= 2 && len(loc2) >= 2 {
- pageNumberSubstring :=
string(content[loc1[0]:loc1[1]])
- pageNumberString :=
strings.Replace(pageNumberSubstring, `page=`, ``, 1)
- pageNameSubstring :=
string(content[loc2[0]:loc2[1]])
- pageNameString :=
strings.Replace(pageNameSubstring, `rel="`, ``, 1)
-
- pageNumberInt, convertErr :=
ConvertStringToInt(pageNumberString)
- if convertErr != nil {
- return result, convertErr
- }
- switch pageNameString {
- case "next":
- result.Next = pageNumberInt
-
- case "first":
- result.First = pageNumberInt
-
- case "last":
- result.Last = pageNumberInt
-
- case "prev":
- result.Prev = pageNumberInt
- }
-
- } else {
- return result, errors.Default.New("parsed
string values aren't long enough")
- }
- }
- return result, nil
- } else {
- return result, errors.Default.New("the link string provided is
invalid. There is likely no next page of data to fetch")
- }
-}
-
-func GetIssueIdByIssueUrl(s string) (int, errors.Error) {
- regex := regexp.MustCompile(`.*/issues/(\d+)`)
- groups := regex.FindStringSubmatch(s)
- if len(groups) > 0 {
- return ConvertStringToInt(groups[1])
- } else {
- return 0, errors.Default.New("invalid issue url")
- }
-}
