This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch release-v0.11-hotfix
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.11-hotfix by this
push:
new fe4e7890 fix: ignore 404 while collecting jira issue worklog
fe4e7890 is described below
commit fe4e789020925fa0ecc6da9f9051a582249fd25e
Author: zhangliang <[email protected]>
AuthorDate: Mon Jun 20 21:02:28 2022 +0800
fix: ignore 404 while collecting jira issue worklog
---
plugins/helper/api_async_client.go | 7 ++++++-
plugins/helper/api_client.go | 6 ++++++
plugins/helper/api_collector.go | 20 ++++++++++++++++++--
plugins/jira/tasks/api_client.go | 10 ++++++++++
plugins/jira/tasks/changelog_collector.go | 1 +
plugins/jira/tasks/remotelink_collector.go | 1 +
plugins/jira/tasks/worklog_collector.go | 1 +
7 files changed, 43 insertions(+), 3 deletions(-)
diff --git a/plugins/helper/api_async_client.go
b/plugins/helper/api_async_client.go
index 20d4f894..b18be35c 100644
--- a/plugins/helper/api_async_client.go
+++ b/plugins/helper/api_async_client.go
@@ -139,7 +139,11 @@ func (apiClient *ApiAsyncClient) DoAsync(
res.Body = io.NopCloser(bytes.NewBuffer(body))
}
}
-
+ if err == ErrIgnoreAndContinue {
+ // make sure defer func got be executed
+ err = nil
+ return nil
+ }
// check
needRetry := false
if err != nil {
@@ -198,6 +202,7 @@ type RateLimitedApiClient interface {
GetQps() float64
Add(delta int)
Done()
+ SetAfterFunction(callback ApiClientAfterResponse)
}
var _ RateLimitedApiClient = (*ApiAsyncClient)(nil)
diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go
index 33b7ef7a..4b941af8 100644
--- a/plugins/helper/api_client.go
+++ b/plugins/helper/api_client.go
@@ -21,6 +21,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
@@ -38,6 +39,8 @@ import (
type ApiClientBeforeRequest func(req *http.Request) error
type ApiClientAfterResponse func(res *http.Response) error
+var ErrIgnoreAndContinue = errors.New("ignore and continue")
+
// ApiClient is designed for simple api requests
type ApiClient struct {
client *http.Client
@@ -220,6 +223,9 @@ func (apiClient *ApiClient) Do(
// after recieve
if apiClient.afterReponse != nil {
err = apiClient.afterReponse(res)
+ if err == ErrIgnoreAndContinue {
+ return res, err
+ }
if err != nil {
res.Body.Close()
return nil, err
diff --git a/plugins/helper/api_collector.go b/plugins/helper/api_collector.go
index 23cbb099..ad057e6b 100644
--- a/plugins/helper/api_collector.go
+++ b/plugins/helper/api_collector.go
@@ -81,6 +81,7 @@ type ApiCollectorArgs struct {
GetTotalPages func(res *http.Response, args *ApiCollectorArgs) (int,
error)
Concurrency int
ResponseParser func(res *http.Response) ([]json.RawMessage, error)
+ AfterResponse ApiClientAfterResponse
}
type ApiCollector struct {
@@ -118,11 +119,22 @@ func NewApiCollector(args ApiCollectorArgs)
(*ApiCollector, error) {
if args.Concurrency < 1 {
args.Concurrency = 1
}
- return &ApiCollector{
+ apicllector := &ApiCollector{
RawDataSubTask: rawDataSubTask,
args: &args,
urlTemplate: tpl,
- }, nil
+ }
+ if args.AfterResponse != nil {
+ apicllector.SetAfterResponse(args.AfterResponse)
+ } else {
+ apicllector.SetAfterResponse(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed,
please check your AccessToken")
+ }
+ return nil
+ })
+ }
+ return apicllector, nil
}
// Start collection
@@ -284,6 +296,10 @@ func (collector *ApiCollector) generateUrl(pager *Pager,
input interface{}) (str
return buf.String(), nil
}
+func (collector *ApiCollector) SetAfterResponse(f ApiClientAfterResponse) {
+ collector.args.ApiClient.SetAfterFunction(f)
+}
+
// stepFetch collect pages synchronously. In practice, several stepFetch
running concurrently, we could stop all of them by calling `cancel`.
func (collector *ApiCollector) stepFetch(ctx context.Context, cancel func(),
reqData RequestData) error {
// channel `c` is used to make sure fetchAsync is called serially
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index 4896c05a..7432ca04 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -86,3 +86,13 @@ func GetJiraServerInfo(client *helper.ApiAsyncClient)
(*models.JiraServerInfo, i
}
return serverInfo, res.StatusCode, nil
}
+
+func ignoreHTTPStatus404(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your
AccessToken")
+ }
+ if res.StatusCode == http.StatusNotFound {
+ return helper.ErrIgnoreAndContinue
+ }
+ return nil
+}
diff --git a/plugins/jira/tasks/changelog_collector.go
b/plugins/jira/tasks/changelog_collector.go
index bd3f37bf..2f13774f 100644
--- a/plugins/jira/tasks/changelog_collector.go
+++ b/plugins/jira/tasks/changelog_collector.go
@@ -98,6 +98,7 @@ func CollectChangelogs(taskCtx core.SubTaskContext) error {
}
return data.Values, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
diff --git a/plugins/jira/tasks/remotelink_collector.go
b/plugins/jira/tasks/remotelink_collector.go
index a38d8ba1..102805d5 100644
--- a/plugins/jira/tasks/remotelink_collector.go
+++ b/plugins/jira/tasks/remotelink_collector.go
@@ -96,6 +96,7 @@ func CollectRemotelinks(taskCtx core.SubTaskContext) error {
}
return result, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
diff --git a/plugins/jira/tasks/worklog_collector.go
b/plugins/jira/tasks/worklog_collector.go
index 70e873d7..e0ad9da3 100644
--- a/plugins/jira/tasks/worklog_collector.go
+++ b/plugins/jira/tasks/worklog_collector.go
@@ -79,6 +79,7 @@ func CollectWorklogs(taskCtx core.SubTaskContext) error {
}
return data.Worklogs, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
logger.Error("collect board error:", err)
