keon94 commented on code in PR #4626:
URL:
https://github.com/apache/incubator-devlake/pull/4626#discussion_r1141611995
##########
backend/helpers/pluginhelper/api/api_collector_with_state.go:
##########
@@ -258,27 +265,31 @@ type FinalizableApiCollectorArgs struct {
CollectUnfinishedDetails FinalizableApiCollectorDetailArgs
}
+// FinalizableApiCollectorCommonArgs is the common arguments for both list and
detail collectors
+// Note that all request-related arguments would be called or utilized before
any response-related arguments
type FinalizableApiCollectorCommonArgs struct {
- UrlTemplate string `comment:"GoTemplate for API url"`
- Query func(reqData *RequestData, createdAfter *time.Time)
(url.Values, errors.Error)
- Header func(reqData *RequestData, createdAfter *time.Time)
(http.Header, errors.Error)
- MinTickInterval *time.Duration
- ResponseParser func(res *http.Response) ([]json.RawMessage,
errors.Error)
- AfterResponse common.ApiClientAfterResponse
- RequestBody func(reqData *RequestData) map[string]interface{}
- Method string
+ UrlTemplate string
// required, url path template for the request, e.g.
repos/{{ .Params.Name }}/pulls
+ Method string
// optional, request method, e.g. GET(default), POST,
PUT, DELETE
+ Query func(reqData *RequestData, createdAfter *time.Time)
(url.Values, errors.Error) // optional, build query params for the request
+ Header func(reqData *RequestData, createdAfter *time.Time)
(http.Header, errors.Error) // optional, build header for the request
+ RequestBody func(reqData *RequestData) map[string]interface{}
// optional, build request body for the request if the
Method set to POST or PUT
+ MinTickInterval *time.Duration
// optional, minimum interval between two requests,
some endpoints might have a more conservative rate limit than others within the
same instance, you can mitigate this by setting a higher MinTickInterval to
override the connection level rate limit.
+ AfterResponse common.ApiClientAfterResponse
// optional, hook to run after each response, would be
called before the ResponseParser
+ ResponseParser func(res *http.Response) ([]json.RawMessage,
errors.Error) // required, parse the response body and
return a list of entities
}
+
+// FinalizableApiCollectorListArgs is the arguments for the list collector
type FinalizableApiCollectorListArgs struct {
- // optional, leave it be `nil` if API supports filtering by created
date (Don't forget to set the Query)
- GetCreated func(item json.RawMessage) (time.Time, errors.Error)
FinalizableApiCollectorCommonArgs
- Concurrency int
- PageSize int
- GetNextPageCustomData func(prevReqData *RequestData, prevPageResponse
*http.Response) (interface{}, errors.Error)
- // need to consider the data missing problem: what if new data gets
created during collection?
- // GetTotalPages func(res *http.Response, args
*ApiCollectorArgs) (int, errors.Error)
+ GetCreated func(item json.RawMessage) (time.Time,
errors.Error) // optional, leave it be
`nil` if API supports filtering by updated date (Don't forget to set the Query)
+ PageSize int
// required, number of records per
page
+ Concurrency int
// required for Undetermined
Strategy, number of concurrent requests
+ GetNextPageCustomData func(prevReqData *RequestData, prevPageResponse
*http.Response) (interface{}, errors.Error) // required for Sequential
Strategy, to extract the next page cursor from the given response
+ GetTotalPages func(res *http.Response, args *ApiCollectorArgs)
(int, errors.Error) // required for Determined Strategy,
to extract the total number of pages from the given response
}
+
+// FinalizableApiCollectorDetailArgs is the arguments for the detail collector
type FinalizableApiCollectorDetailArgs struct {
FinalizableApiCollectorCommonArgs
- BuildInputIterator func() (Iterator, errors.Error)
+ BuildInputIterator func() (Iterator, errors.Error) // required, build
an iterator which would iterate all unfinished records in the database so they
can be recollected and updated
Review Comment:
Let's add an extra sentence to explain which function would be using the
elements of the iterator.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
