This is an automated email from the ASF dual-hosted git repository. ka94 pushed a commit to branch release-v0.17 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0bc680ee434a34663fef326593a613ab1ef8463f Author: Keon Amini <[email protected]> AuthorDate: Tue May 16 12:29:05 2023 -0700 fix: PagerDuty now limits to 10k latest incidents per API spec, and pulls based on service ids (#5196) --- backend/helpers/pluginhelper/api/api_async_client.go | 9 +++++---- backend/plugins/pagerduty/tasks/incidents_collector.go | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/helpers/pluginhelper/api/api_async_client.go b/backend/helpers/pluginhelper/api/api_async_client.go index e45794c44..715245d64 100644 --- a/backend/helpers/pluginhelper/api/api_async_client.go +++ b/backend/helpers/pluginhelper/api/api_async_client.go @@ -181,13 +181,14 @@ func (apiClient *ApiAsyncClient) DoAsync( // check needRetry := false + errMessage := "unknown" if err != nil { needRetry = true + errMessage = err.Error() } else if res.StatusCode >= HttpMinStatusRetryCode { needRetry = true - err = errors.HttpStatus(res.StatusCode).New( - fmt.Sprintf("Http DoAsync error calling [%s %s]. Response: %s", method, path, string(respBody)), - ) + errMessage = fmt.Sprintf("Http DoAsync error calling [%s %s]. Response: %s", method, path, string(respBody)) + err = errors.HttpStatus(res.StatusCode).New(errMessage) } // if it needs retry, check and retry @@ -205,7 +206,7 @@ func (apiClient *ApiAsyncClient) DoAsync( } if err != nil { - err = errors.Default.Wrap(err, fmt.Sprintf("retry exceeded %d times calling %s", retry, path)) + err = errors.Default.Wrap(err, fmt.Sprintf("Retry exceeded %d times calling %s. The last error was: %s", retry, path, errMessage)) apiClient.logger.Error(err, "") return errors.Convert(err) } diff --git a/backend/plugins/pagerduty/tasks/incidents_collector.go b/backend/plugins/pagerduty/tasks/incidents_collector.go index 883f071bf..e95de2e6d 100644 --- a/backend/plugins/pagerduty/tasks/incidents_collector.go +++ b/backend/plugins/pagerduty/tasks/incidents_collector.go @@ -73,13 +73,12 @@ func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error { TimeAfter: data.TimeAfter, CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{ PageSize: 100, - GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { - paging := pagingInfo{} - err := api.UnmarshalResponse(res, &paging) - if err != nil { - return 0, errors.BadInput.Wrap(err, "failed to determined paging count") + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + pager := prevReqData.Pager + if pager.Skip+pager.Size >= 10_000 { // API limit. Can't exceed this or it'll error out + return nil, api.ErrFinishCollect } - return *paging.Total, nil + return nil, nil }, FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ UrlTemplate: "incidents", @@ -97,7 +96,8 @@ func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error { } else { query.Set("date_range", "all") } - query.Set("sort_by", "created_at:desc") + query.Set("service_ids[]", data.Options.ServiceId) + query.Set("sort_by", "created_at:desc") // the newest entries will be fetched first query.Set("limit", fmt.Sprintf("%d", reqData.Pager.Size)) query.Set("offset", fmt.Sprintf("%d", reqData.Pager.Skip)) query.Set("total", "true")
