This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new c5905f75f fix(github): bisect adaptive time windows for workflow runs 
40k pagination cap (#8844)
c5905f75f is described below

commit c5905f75fadf6a6b5aecd686c0118ed3e774d174
Author: Tomoya Kawaguchi <[email protected]>
AuthorDate: Sat Apr 25 22:00:16 2026 +0900

    fix(github): bisect adaptive time windows for workflow runs 40k pagination 
cap (#8844)
    
    GitHub's /actions/runs enforces a 40k cap in unfiltered mode and a 
1,000-item
    cap per filtered search, making any repo with >40k workflow runs 
uncollectable.
    Switch to filtered mode and recursively bisect time windows at 
integer-second
    midpoints, probing via SubmitBlocking to share rate-limit with the main
    collector and feeding leaves to a single ApiCollector so raw-table Delete
    fires only once. See PR description for the full design rationale.
    
    Closes #8842
    
    Signed-off-by: yamoyamoto <[email protected]>
---
 backend/plugins/github/tasks/cicd_run_collector.go | 324 +++++++++++---
 .../github/tasks/cicd_run_collector_test.go        | 490 +++++++++++++++++++++
 2 files changed, 750 insertions(+), 64 deletions(-)

diff --git a/backend/plugins/github/tasks/cicd_run_collector.go 
b/backend/plugins/github/tasks/cicd_run_collector.go
index 68bd0aa15..44c816ca2 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -25,7 +25,7 @@ import (
        "time"
 
        "github.com/apache/incubator-devlake/core/errors"
-       "github.com/apache/incubator-devlake/core/models/common"
+       "github.com/apache/incubator-devlake/core/log"
        "github.com/apache/incubator-devlake/core/plugin"
        helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 )
@@ -36,14 +36,20 @@ func init() {
 
 const RAW_RUN_TABLE = "github_api_runs"
 
-// Although the API accepts a maximum of 100 entries per page, sometimes
-// the response body is too large which would lead to request failures
-// https://github.com/apache/incubator-devlake/issues/3199
+// PAGE_SIZE is below GitHub's 100-item max to avoid oversized response bodies 
(#3199).
 const PAGE_SIZE = 30
 
-type SimpleGithubApiJob struct {
-       ID        int64
-       CreatedAt common.Iso8601Time `json:"created_at"`
+// FILTERED_SEARCH_CAP is GitHub's per-query item cap for `/actions/runs` in 
filtered mode
+// (`created=<from>..<to>`); exceeding it triggers HTTP 422. See #8842.
+const FILTERED_SEARCH_CAP = 1000
+
+// githubTimeLayout is the ISO8601 format GitHub expects in the `created` 
filter.
+const githubTimeLayout = "2006-01-02T15:04:05Z"
+
+// TimeWindow is an inclusive-both-ends range for the `/actions/runs` 
`created=<from>..<to>` query.
+type TimeWindow struct {
+       From time.Time
+       To   time.Time
 }
 
 var CollectRunsMeta = plugin.SubTaskMeta{
@@ -56,75 +62,265 @@ var CollectRunsMeta = plugin.SubTaskMeta{
        ProductTables:    []string{RAW_RUN_TABLE},
 }
 
+// CollectRuns collects GitHub Workflow Runs into the raw table, working 
around the two
+// pagination caps of `/actions/runs`: 40k items in unfiltered mode and 1000 
items per
+// filtered window. It probes each candidate window with `per_page=1`, bisects 
recursively
+// until every leaf is under FILTERED_SEARCH_CAP, then feeds the leaves to a 
single
+// ApiCollector so the raw table is truncated only once per fullsync.
 func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
        data := taskCtx.GetData().(*GithubTaskData)
-       log := taskCtx.GetLogger()
-       collector, err := 
helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
-               RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
-                       Ctx: taskCtx,
-                       Params: GithubApiParams{
-                               ConnectionId: data.Options.ConnectionId,
-                               Name:         data.Options.Name,
-                       },
-                       Table: RAW_RUN_TABLE,
+       logger := taskCtx.GetLogger()
+
+       manager, err := 
helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{
+               Ctx: taskCtx,
+               Params: GithubApiParams{
+                       ConnectionId: data.Options.ConnectionId,
+                       Name:         data.Options.Name,
                },
-               ApiClient: data.ApiClient,
-               CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
-                       PageSize:    PAGE_SIZE,
-                       Concurrency: 10,
-                       FinalizableApiCollectorCommonArgs: 
helper.FinalizableApiCollectorCommonArgs{
-                               UrlTemplate: "repos/{{ .Params.Name 
}}/actions/runs",
-                               Query: func(reqData *helper.RequestData, 
createdAfter *time.Time) (url.Values, errors.Error) {
-                                       query := url.Values{}
-                                       // GitHub API returns only the first 34 
pages (with a size of 30) when specifying status=compleleted, try the following 
API request to verify the problem.
-                                       // 
https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
-                                       // query.Set("status", "completed")
-                                       query.Set("page", fmt.Sprintf("%v", 
reqData.Pager.Page))
-                                       query.Set("per_page", fmt.Sprintf("%v", 
reqData.Pager.Size))
-                                       return query, nil
-                               },
-                               ResponseParser: func(res *http.Response) 
([]json.RawMessage, errors.Error) {
-                                       body := &GithubRawRunsResult{}
-                                       err := helper.UnmarshalResponse(res, 
body)
-                                       if err != nil {
-                                               return nil, err
-                                       }
-                                       if len(body.WorkflowRuns) == 0 {
-                                               return nil, nil
-                                       }
-                                       // filter out the runs that are not 
completed
-                                       filteredRuns := make([]json.RawMessage, 
0)
-                                       for _, run := range body.WorkflowRuns {
-                                               if run.Status == "completed" {
-                                                       runJSON, err := 
json.Marshal(run)
-                                                       if err != nil {
-                                                               return nil, 
errors.Convert(err)
-                                                       }
-                                                       filteredRuns = 
append(filteredRuns, json.RawMessage(runJSON))
-                                               } else {
-                                                       log.Info("Skipping 
run{id: %d, number: %d} with status %s", run.ID, run.RunNumber, run.Status)
-                                               }
-                                       }
-                                       return filteredRuns, nil
-                               },
-                       },
-                       GetCreated: func(item json.RawMessage) (time.Time, 
errors.Error) {
-                               pj := &SimpleGithubApiJob{}
-                               err := json.Unmarshal(item, pj)
+               Table: RAW_RUN_TABLE,
+       })
+       if err != nil {
+               return err
+       }
+
+       // Normalize both bounds to second precision so `created=<from>..<to>` 
queries and the
+       // persisted LatestSuccessStart share the same boundary; without this, 
every incremental
+       // sync would re-fetch up to 1s of overlap. For incremental syncs we 
also advance
+       // `windowStart` past the previously collected second 
(inclusive-both-ends), while
+       // fullsync + TimeAfter keeps the user-specified bound inclusive.
+       createdAfter := manager.GetSince()
+       untilPtr := manager.GetUntil()
+       *untilPtr = untilPtr.Truncate(time.Second)
+       until := *untilPtr
+
+       var windowStart time.Time
+       if createdAfter != nil {
+               windowStart = createdAfter.Truncate(time.Second)
+               if manager.IsIncremental() {
+                       windowStart = windowStart.Add(time.Second)
+               }
+       } else {
+               // 2018-01-01 conservatively predates GitHub Actions' late-2019 
GA.
+               windowStart = time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)
+       }
+
+       logger.Info("cicd_run_collector: collecting workflow runs in [%s, %s] 
(incremental=%v)",
+               windowStart.Format(githubTimeLayout),
+               until.Format(githubTimeLayout),
+               manager.IsIncremental())
+
+       leafWindows, err := newLeafWindowBuilder(taskCtx, 
data).build(windowStart, until)
+       if err != nil {
+               return err
+       }
+       logger.Info("cicd_run_collector: built %d leaf windows for collection", 
len(leafWindows))
+
+       if err := registerCollectorForLeafWindows(manager, data.ApiClient, 
leafWindows); err != nil {
+               return err
+       }
+
+       return manager.Execute()
+}
+
+// buildRunsQuery assembles the filtered-mode query for a single leaf 
TimeWindow.
+// Shared between registerCollectorForLeafWindows and tests.
+func buildRunsQuery(reqData *helper.RequestData) (url.Values, errors.Error) {
+       w, ok := reqData.Input.(*TimeWindow)
+       if !ok || w == nil {
+               return nil, errors.Default.New("cicd_run_collector: Input is 
not *TimeWindow")
+       }
+       q := url.Values{}
+       q.Set("created", fmt.Sprintf("%s..%s",
+               w.From.UTC().Format(githubTimeLayout),
+               w.To.UTC().Format(githubTimeLayout)))
+       q.Set("page", fmt.Sprintf("%d", reqData.Pager.Page))
+       q.Set("per_page", fmt.Sprintf("%d", reqData.Pager.Size))
+       return q, nil
+}
+
+// registerCollectorForLeafWindows wires a single ApiCollector whose Input 
iterator feeds the
+// leaf TimeWindows.
+func registerCollectorForLeafWindows(
+       manager *helper.StatefulApiCollector,
+       apiClient helper.RateLimitedApiClient,
+       leafWindows []TimeWindow,
+) errors.Error {
+       iterator := helper.NewQueueIterator()
+       for i := range leafWindows {
+               w := leafWindows[i]
+               iterator.Push(&w)
+       }
+       return manager.InitCollector(helper.ApiCollectorArgs{
+               ApiClient:   apiClient,
+               Input:       iterator,
+               UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
+               Query:       buildRunsQuery,
+               PageSize:    PAGE_SIZE,
+               Concurrency: 10,
+               ResponseParser: func(res *http.Response) ([]json.RawMessage, 
errors.Error) {
+                       body := &GithubRawRunsResult{}
+                       if err := helper.UnmarshalResponse(res, body); err != 
nil {
+                               return nil, err
+                       }
+                       if len(body.WorkflowRuns) == 0 {
+                               return nil, nil
+                       }
+                       // Range is already bounded in filtered mode; only keep 
completed runs.
+                       filtered := make([]json.RawMessage, 0, 
len(body.WorkflowRuns))
+                       for _, run := range body.WorkflowRuns {
+                               if run.Status != "completed" {
+                                       continue
+                               }
+                               runJSON, err := json.Marshal(run)
                                if err != nil {
-                                       return time.Time{}, 
errors.BadInput.Wrap(err, "failed to unmarshal github run")
+                                       return nil, errors.Convert(err)
                                }
-                               return pj.CreatedAt.ToTime(), nil
-                       },
+                               filtered = append(filtered, 
json.RawMessage(runJSON))
+                       }
+                       return filtered, nil
                },
        })
+}
+
+// probeFunc signature matches defaultProbeTotalCount so tests can inject a 
fake.
+type probeFunc func(taskCtx plugin.SubTaskContext, data *GithubTaskData, from, 
to time.Time) (int, bool, errors.Error)
+
+// leafWindowBuilder recursively bisects a [from, to] range until every leaf 
window fits under
+// FILTERED_SEARCH_CAP. The probe function is a field so tests can inject a 
fake without
+// mutating package-level state.
+type leafWindowBuilder struct {
+       taskCtx plugin.SubTaskContext
+       data    *GithubTaskData
+       probe   probeFunc
+       logger  log.Logger
+}
+
+func newLeafWindowBuilder(taskCtx plugin.SubTaskContext, data *GithubTaskData) 
*leafWindowBuilder {
+       return &leafWindowBuilder{
+               taskCtx: taskCtx,
+               data:    data,
+               probe:   defaultProbeTotalCount,
+               logger:  taskCtx.GetLogger(),
+       }
+}
+
+// build recursively bisects [from, to] until every leaf has total_count < 
FILTERED_SEARCH_CAP
+// (or the window is a single-second bucket that cannot be split further). 
Empty windows are
+// dropped.
+//
+// Boundary policy (non-overlapping, full coverage at second precision):
+//   - left:  created=<from>..<mid>
+//   - right: created=<mid+1s>..<to>
+//
+// Bisection is done on integer Unix seconds because GitHub's `created` filter 
is
+// second-precision; a single-second bucket (from.Unix() == to.Unix()) is the 
smallest
+// indivisible unit.
+func (b *leafWindowBuilder) build(from, to time.Time) ([]TimeWindow, 
errors.Error) {
+       if !from.Before(to) && !from.Equal(to) {
+               return nil, nil
+       }
 
+       total, is422, err := b.probe(b.taskCtx, b.data, from, to)
        if err != nil {
-               return err
+               return nil, err
+       }
+
+       if total == 0 && !is422 {
+               return nil, nil
        }
 
-       return collector.Execute()
+       if total >= FILTERED_SEARCH_CAP || is422 {
+               fromSec := from.UTC().Unix()
+               toSec := to.UTC().Unix()
+               if fromSec == toSec {
+                       return nil, errors.Default.New(fmt.Sprintf(
+                               "cicd_run_collector: %d runs within a single 
1-second bucket at %s; cannot bisect further. "+
+                                       "Filtered GitHub search caps at %d 
items per window, so some runs would be missed. "+
+                                       "Refusing to advance collector state.",
+                               total, from.UTC().Format(time.RFC3339), 
FILTERED_SEARCH_CAP,
+                       ))
+               }
+               if b.logger != nil {
+                       b.logger.Debug("cicd_run_collector: bisecting [%s, %s] 
(total=%d, is422=%v)",
+                               from.Format(githubTimeLayout),
+                               to.Format(githubTimeLayout),
+                               total, is422)
+               }
+               midSec := (fromSec + toSec) / 2
+               leftTo := time.Unix(midSec, 0).UTC()
+               rightFrom := leftTo.Add(time.Second)
+               left, err := b.build(from, leftTo)
+               if err != nil {
+                       return nil, err
+               }
+               right, err := b.build(rightFrom, to)
+               if err != nil {
+                       return nil, err
+               }
+               return append(left, right...), nil
+       }
 
+       return []TimeWindow{{From: from, To: to}}, nil
+}
+
+// defaultProbeTotalCount issues a filtered-mode GET with per_page=1 to learn 
total_count
+// (or detect 422) cheaply. It runs under SubmitBlocking so it shares the 
rate-limit budget
+// with the main collector; DoGetAsync is avoided because that path errors on 
>=400 before
+// the callback, which would hide the 422 we use as a bisection signal.
+func defaultProbeTotalCount(
+       taskCtx plugin.SubTaskContext,
+       data *GithubTaskData,
+       from time.Time,
+       to time.Time,
+) (int, bool, errors.Error) {
+       q := url.Values{}
+       q.Set("per_page", "1")
+       q.Set("page", "1")
+       q.Set("created", fmt.Sprintf("%s..%s",
+               from.UTC().Format(githubTimeLayout),
+               to.UTC().Format(githubTimeLayout)))
+       path := fmt.Sprintf("repos/%s/actions/runs", data.Options.Name)
+
+       var total int
+       var is422 bool
+       var innerErr errors.Error
+       data.ApiClient.SubmitBlocking(func() errors.Error {
+               res, getErr := data.ApiClient.Get(path, q, nil)
+               // If a sibling subtask installed an AfterResponse hook that 
maps 422 ->
+               // helper.ErrIgnoreAndContinue, ApiClient.Do already closed 
res.Body before returning
+               // the sentinel (api_client.go L389). Recover the 422 signal 
here without double-closing.
+               // Sentinel comparison uses `==` to stay consistent with every 
other ErrIgnoreAndContinue
+               // call site in devlake (api_client.go:389, 
api_async_client.go:165, etc.).
+               if getErr == helper.ErrIgnoreAndContinue && res != nil && 
res.StatusCode == http.StatusUnprocessableEntity {
+                       is422 = true
+                       return nil
+               }
+               if getErr != nil {
+                       innerErr = getErr
+                       return nil
+               }
+               if res.StatusCode == http.StatusUnprocessableEntity {
+                       if res.Body != nil {
+                               _ = res.Body.Close()
+                       }
+                       is422 = true
+                       return nil
+               }
+               body := &GithubRawRunsResult{}
+               if e := helper.UnmarshalResponse(res, body); e != nil {
+                       innerErr = e
+                       return nil
+               }
+               total = body.TotalCount
+               return nil
+       })
+       if err := data.ApiClient.WaitAsync(); err != nil {
+               return 0, false, err
+       }
+       if innerErr != nil {
+               return 0, false, innerErr
+       }
+       return total, is422, nil
 }
 
 type GithubRawRunsResult struct {
diff --git a/backend/plugins/github/tasks/cicd_run_collector_test.go 
b/backend/plugins/github/tasks/cicd_run_collector_test.go
new file mode 100644
index 000000000..d69b5c73b
--- /dev/null
+++ b/backend/plugins/github/tasks/cicd_run_collector_test.go
@@ -0,0 +1,490 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+       "bytes"
+       "encoding/json"
+       "io"
+       "net/http"
+       "net/url"
+       "strings"
+       "sync/atomic"
+       "testing"
+       "time"
+
+       "github.com/apache/incubator-devlake/core/errors"
+       "github.com/apache/incubator-devlake/core/plugin"
+       helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+       "github.com/apache/incubator-devlake/helpers/unithelper"
+       mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
+       mockapi 
"github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api"
+
+       "github.com/stretchr/testify/assert"
+       "github.com/stretchr/testify/mock"
+       "github.com/stretchr/testify/require"
+)
+
+// newTestBuilder constructs a leafWindowBuilder with a stubbed probe for unit 
testing.
+func newTestBuilder(probe probeFunc) *leafWindowBuilder {
+       mockDal := new(mockdal.Dal)
+       return &leafWindowBuilder{
+               taskCtx: unithelper.DummySubTaskContext(mockDal),
+               data:    &GithubTaskData{Options: &GithubOptions{Name: "o/r"}},
+               probe:   probe,
+       }
+}
+
+// 
------------------------------------------------------------------------------------------
+// leafWindowBuilder.build unit tests
+// 
------------------------------------------------------------------------------------------
+
+func TestCicdRunBuildLeafWindows_SingleWindowUnderCap(t *testing.T) {
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, _, 
_ time.Time) (int, bool, errors.Error) {
+               return 500, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 1)
+       assert.True(t, leaves[0].From.Equal(from))
+       assert.True(t, leaves[0].To.Equal(to))
+}
+
+func TestCicdRunBuildLeafWindows_EmptyWindow(t *testing.T) {
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, _, 
_ time.Time) (int, bool, errors.Error) {
+               return 0, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       assert.Empty(t, leaves)
+}
+
+// TestCicdRunBuildLeafWindows_InvertedWindowReturnsNoLeaves pins the 
entry-guard contract:
+// when `from > to` (rapid re-sync edge case), build must short-circuit to 
(nil, nil)
+// without probing so no GitHub round-trip is paid on a no-op sync.
+func TestCicdRunBuildLeafWindows_InvertedWindowReturnsNoLeaves(t *testing.T) {
+       var probeCalls int32
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, _, 
_ time.Time) (int, bool, errors.Error) {
+               atomic.AddInt32(&probeCalls, 1)
+               return 0, false, nil
+       })
+
+       to := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       from := to.Add(2 * time.Second)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       assert.Nil(t, leaves)
+       assert.Equal(t, int32(0), atomic.LoadInt32(&probeCalls))
+}
+
+func TestCicdRunBuildLeafWindows_BisectsOnCap(t *testing.T) {
+       var callCount int32
+       fullWindow := time.Date(2024, 2, 1, 0, 0, 0, 0, 
time.UTC).Sub(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC))
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
to time.Time) (int, bool, errors.Error) {
+               atomic.AddInt32(&callCount, 1)
+               if to.Sub(f) >= fullWindow {
+                       return 1000, false, nil
+               }
+               return 500, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+       assert.GreaterOrEqual(t, atomic.LoadInt32(&callCount), int32(3))
+       assert.True(t, leaves[0].From.Equal(from))
+       assert.True(t, leaves[1].To.Equal(to))
+}
+
+func TestCicdRunBuildLeafWindows_BisectsOn422(t *testing.T) {
+       fullWindow := time.Date(2024, 2, 1, 0, 0, 0, 0, 
time.UTC).Sub(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC))
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
to time.Time) (int, bool, errors.Error) {
+               if to.Sub(f) >= fullWindow {
+                       return 0, true, nil
+               }
+               return 100, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+}
+
+func TestCicdRunBuildLeafWindows_MinWindowReturnsError(t *testing.T) {
+       // Only a single-second bucket (from.Unix() == to.Unix()) is truly 
unbisectable; a 1s-wide
+       // window bisects into [T, T] and [T+1, T+1] and the recursive call on 
the inner bucket
+       // surfaces the error.
+       t.Run("zero-width saturated window -> error", func(t *testing.T) {
+               b := newTestBuilder(func(_ plugin.SubTaskContext, _ 
*GithubTaskData, _, _ time.Time) (int, bool, errors.Error) {
+                       return 1000, false, nil
+               })
+
+               from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+               to := from
+
+               leaves, err := b.build(from, to)
+               assert.Nil(t, leaves)
+               require.NotNil(t, err)
+               assert.Contains(t, err.Error(), "single 1-second bucket")
+               assert.Contains(t, err.Error(), "Refusing to advance collector 
state")
+       })
+
+       t.Run("1s-wide saturated window bisects then fails on inner 
single-second bucket", func(t *testing.T) {
+               b := newTestBuilder(func(_ plugin.SubTaskContext, _ 
*GithubTaskData, _, _ time.Time) (int, bool, errors.Error) {
+                       return 1000, false, nil
+               })
+
+               from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+               to := from.Add(time.Second)
+
+               leaves, err := b.build(from, to)
+               assert.Nil(t, leaves)
+               require.NotNil(t, err)
+               assert.Contains(t, err.Error(), "single 1-second bucket")
+       })
+}
+
+func TestCicdRunBuildLeafWindows_MinWindowOn422ReturnsError(t *testing.T) {
+       t.Run("zero-width 422 window -> error", func(t *testing.T) {
+               b := newTestBuilder(func(_ plugin.SubTaskContext, _ 
*GithubTaskData, _, _ time.Time) (int, bool, errors.Error) {
+                       return 0, true, nil
+               })
+
+               from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+               to := from
+
+               leaves, err := b.build(from, to)
+               assert.Nil(t, leaves)
+               require.NotNil(t, err)
+               assert.Contains(t, err.Error(), "single 1-second bucket")
+       })
+
+       t.Run("1s-wide 422 window bisects then fails on inner single-second 
bucket", func(t *testing.T) {
+               b := newTestBuilder(func(_ plugin.SubTaskContext, _ 
*GithubTaskData, _, _ time.Time) (int, bool, errors.Error) {
+                       return 0, true, nil
+               })
+
+               from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+               to := from.Add(time.Second)
+
+               leaves, err := b.build(from, to)
+               assert.Nil(t, leaves)
+               require.NotNil(t, err)
+               assert.Contains(t, err.Error(), "single 1-second bucket")
+       })
+}
+
+func TestCicdRunBuildLeafWindows_NoRightHalfEmptyAtSubSecondWidths(t 
*testing.T) {
+       // Regression guard: integer-second bisection keeps both halves inside 
[from, to] at
+       // sub-second-tail widths; the old duration-based bisector failed with 
"right half empty".
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := from.Add(1500 * time.Millisecond)
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
tt time.Time) (int, bool, errors.Error) {
+               if f.Equal(from) && tt.Equal(to) {
+                       return 1000, false, nil
+               }
+               return 500, false, nil
+       })
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+       assert.Equal(t, from.Unix(), leaves[0].From.Unix())
+       assert.Equal(t, from.Unix(), leaves[0].To.Unix())
+       assert.Equal(t, from.Unix()+1, leaves[1].From.Unix())
+       assert.True(t, leaves[1].To.Equal(to))
+}
+
+// TestCicdRunBuildLeafWindows_ProbeErrIgnoreAndContinueRecovery pins the 
contract that
+// `build` treats (total=0, is422=true, err=nil) as a valid saturated signal, 
which is what
+// defaultProbeTotalCount emits when a shared ApiAsyncClient's AfterResponse 
hook converts
+// 422 into helper.ErrIgnoreAndContinue.
+func TestCicdRunBuildLeafWindows_ProbeErrIgnoreAndContinueRecovery(t 
*testing.T) {
+       fullWindow := time.Date(2024, 2, 1, 0, 0, 0, 0, 
time.UTC).Sub(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC))
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
to time.Time) (int, bool, errors.Error) {
+               if to.Sub(f) >= fullWindow {
+                       return 0, true, nil
+               }
+               return 200, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+       assert.True(t, leaves[0].From.Equal(from))
+       assert.True(t, leaves[1].To.Equal(to))
+       assert.True(t, leaves[1].From.Equal(leaves[0].To.Add(time.Second)))
+}
+
+func TestCicdRunBuildLeafWindows_ProbeErrorPropagates(t *testing.T) {
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, _, 
_ time.Time) (int, bool, errors.Error) {
+               return 0, false, errors.Default.New("boom")
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       assert.Nil(t, leaves)
+       require.NotNil(t, err)
+       assert.Contains(t, err.Error(), "boom")
+}
+
+func TestCicdRunBuildLeafWindows_BoundaryCases(t *testing.T) {
+       // Boundary total_count values at zero-width (unbisectable 
single-second bucket) and 1s
+       // (bisectable) widths.
+       cases := []struct {
+               total       int
+               is422       bool
+               expectLeaf  bool
+               expectError bool
+               description string
+       }{
+               {0, false, false, false, "zero total -> no leaf"},
+               {1, false, true, false, "1 total -> one leaf"},
+               {999, false, true, false, "999 total -> one leaf"},
+               {1000, false, false, true, "1000 -> error (saturated at 
bucket)"},
+               {1001, false, false, true, "1001 -> error (saturated at 
bucket)"},
+               {0, true, false, true, "422 -> error (saturated at bucket)"},
+       }
+       widths := []struct {
+               name     string
+               duration time.Duration
+       }{
+               {"zero-width (single-second bucket)", 0},
+               {"1s-wide (bisectable at second precision)", time.Second},
+       }
+       for _, w := range widths {
+               for _, c := range cases {
+                       t.Run(w.name+"/"+c.description, func(t *testing.T) {
+                               total := c.total
+                               is422 := c.is422
+                               b := newTestBuilder(func(_ 
plugin.SubTaskContext, _ *GithubTaskData, _, _ time.Time) (int, bool, 
errors.Error) {
+                                       return total, is422, nil
+                               })
+
+                               from := time.Date(2024, 1, 1, 0, 0, 0, 0, 
time.UTC)
+                               to := from.Add(w.duration)
+
+                               leaves, err := b.build(from, to)
+                               if c.expectError {
+                                       require.NotNil(t, err)
+                                       assert.Nil(t, leaves)
+                                       assert.Contains(t, err.Error(), "single 
1-second bucket")
+                                       return
+                               }
+                               require.Nil(t, err)
+                               if c.expectLeaf {
+                                       assert.Len(t, leaves, 1)
+                               } else {
+                                       assert.Empty(t, leaves)
+                               }
+                       })
+               }
+       }
+}
+
+func TestCicdRunBuildLeafWindows_IntegerSecondSplit(t *testing.T) {
+       // Parent [T, T+1s] with total=1000 must split into two 
single-second-bucket leaves
+       // [T, T] and [T+1, T+1], each with total < FILTERED_SEARCH_CAP.
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := from.Add(time.Second)
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
tt time.Time) (int, bool, errors.Error) {
+               if f.Equal(from) && tt.Equal(to) {
+                       return 1000, false, nil
+               }
+               return 500, false, nil
+       })
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+
+       assert.Equal(t, from.Unix(), leaves[0].From.Unix())
+       assert.Equal(t, from.Unix(), leaves[0].To.Unix())
+       assert.Equal(t, from.Unix()+1, leaves[1].From.Unix())
+       assert.Equal(t, from.Unix()+1, leaves[1].To.Unix())
+       assert.True(t, leaves[1].From.Equal(leaves[0].To.Add(time.Second)))
+}
+
+func TestCicdRunBuildLeafWindows_BootstrapFromEpoch(t *testing.T) {
+       // Simulate createdAfter == nil path: caller passes 2018-01-01 as start.
+       full := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).Sub(time.Date(2018, 
1, 1, 0, 0, 0, 0, time.UTC))
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
to time.Time) (int, bool, errors.Error) {
+               if to.Sub(f) >= full/2 {
+                       return 1000, false, nil
+               }
+               return 500, false, nil
+       })
+
+       from := time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       assert.GreaterOrEqual(t, len(leaves), 2)
+}
+
+// 
------------------------------------------------------------------------------------------
+// Boundary (non-overlap) test
+// 
------------------------------------------------------------------------------------------
+
+func TestCicdRunBuildLeafWindows_BoundaryNonOverlapping(t *testing.T) {
+       fullWindow := time.Date(2024, 2, 1, 0, 0, 0, 0, 
time.UTC).Sub(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC))
+       b := newTestBuilder(func(_ plugin.SubTaskContext, _ *GithubTaskData, f, 
to time.Time) (int, bool, errors.Error) {
+               if to.Sub(f) >= fullWindow {
+                       return 1000, false, nil
+               }
+               return 500, false, nil
+       })
+
+       from := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+       to := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)
+
+       leaves, err := b.build(from, to)
+       require.Nil(t, err)
+       require.Len(t, leaves, 2)
+
+       assert.True(t, leaves[1].From.Equal(leaves[0].To.Add(time.Second)),
+               "right.From (%s) must equal left.To+1s (%s)",
+               leaves[1].From.UTC().Format(time.RFC3339),
+               leaves[0].To.Add(time.Second).UTC().Format(time.RFC3339))
+
+       leftStr := leaves[0].To.UTC().Format(githubTimeLayout)
+       rightStr := leaves[1].From.UTC().Format(githubTimeLayout)
+       assert.NotEqual(t, leftStr, rightStr, "boundary timestamps must differ 
by at least 1s")
+}
+
+// 
------------------------------------------------------------------------------------------
+// Query hook window application
+// 
------------------------------------------------------------------------------------------
+
+func TestCicdRunQueryHookAppliesWindow(t *testing.T) {
+       from := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
+       to := time.Date(2024, 6, 7, 8, 9, 10, 0, time.UTC)
+       reqData := &helper.RequestData{
+               Pager: &helper.Pager{Page: 2, Size: 30},
+               Input: &TimeWindow{From: from, To: to},
+       }
+
+       q, err := buildRunsQuery(reqData)
+       require.Nil(t, err)
+       assert.Equal(t, "2024-01-02T03:04:05Z..2024-06-07T08:09:10Z", 
q.Get("created"))
+       assert.Equal(t, "2", q.Get("page"))
+       assert.Equal(t, "30", q.Get("per_page"))
+       assert.True(t, strings.Contains(q.Get("created"), ".."))
+}
+
+// 
------------------------------------------------------------------------------------------
+// Thin integration test
+// Verifies:
+//   - Delete (raw table truncation) is called at most once for a full sync
+//   - DoGetAsync is called at least once per leaf window (concurrency=1, 
PageSize=3)
+// 
------------------------------------------------------------------------------------------
+
+func TestCicdRunRegisterCollectorForLeafWindows_SingleDelete(t *testing.T) {
+       mockDal := new(mockdal.Dal)
+       notFoundErr := errors.Default.New("record not found")
+       mockDal.On("First", mock.Anything, 
mock.Anything).Return(notFoundErr).Once()
+       mockDal.On("IsErrorNotFound", mock.Anything).Return(true).Maybe()
+       mockDal.On("Update", mock.Anything, mock.Anything).Return(nil).Maybe()
+       mockDal.On("AutoMigrate", mock.Anything, 
mock.Anything).Return(nil).Once()
+       mockDal.On("Delete", mock.Anything, mock.Anything, 
mock.Anything).Return(nil).Once()
+       mockDal.On("Create", mock.Anything, mock.Anything).Return(nil).Maybe()
+
+       mockCtx := unithelper.DummySubTaskContext(mockDal)
+
+       var getAsyncCount int32
+       mockApi := new(mockapi.RateLimitedApiClient)
+       mockApi.On("DoGetAsync", mock.Anything, mock.Anything, mock.Anything, 
mock.Anything).Run(func(args mock.Arguments) {
+               atomic.AddInt32(&getAsyncCount, 1)
+               res := &http.Response{
+                       Request: &http.Request{URL: &url.URL{}},
+                       Body:    
io.NopCloser(bytes.NewBufferString(`{"total_count":0,"workflow_runs":[]}`)),
+               }
+               handler := args.Get(3).(plugin.ApiAsyncCallback)
+               _ = handler(res)
+       })
+       mockApi.On("NextTick", mock.Anything).Run(func(args mock.Arguments) {
+               fn := args.Get(0).(func() errors.Error)
+               assert.Nil(t, fn())
+       })
+       mockApi.On("HasError").Return(false)
+       mockApi.On("WaitAsync").Return(nil)
+       mockApi.On("GetAfterFunction").Return(nil)
+       mockApi.On("SetAfterFunction", mock.Anything).Return()
+
+       manager, err := 
helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{
+               Ctx: mockCtx,
+               Params: GithubApiParams{
+                       ConnectionId: 1,
+                       Name:         "apache/incubator-devlake",
+               },
+               Table: RAW_RUN_TABLE,
+       })
+       require.Nil(t, err)
+
+       leaves := []TimeWindow{
+               {From: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), To: 
time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC)},
+               {From: time.Date(2024, 1, 15, 0, 0, 1, 0, time.UTC), To: 
time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)},
+               {From: time.Date(2024, 2, 1, 0, 0, 1, 0, time.UTC), To: 
time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC)},
+       }
+
+       // Force concurrency=1 and PageSize=3 for deterministic DoGetAsync 
counting; share the
+       // production Query hook via buildRunsQuery so changes to it stay 
covered.
+       iterator := helper.NewQueueIterator()
+       for i := range leaves {
+               w := leaves[i]
+               iterator.Push(&w)
+       }
+       err = manager.InitCollector(helper.ApiCollectorArgs{
+               ApiClient:   mockApi,
+               Input:       iterator,
+               UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
+               Query:       buildRunsQuery,
+               PageSize:    3,
+               Concurrency: 1,
+               ResponseParser: func(res *http.Response) ([]json.RawMessage, 
errors.Error) {
+                       return nil, nil
+               },
+       })
+       require.Nil(t, err)
+
+       require.Nil(t, manager.Execute())
+
+       mockDal.AssertExpectations(t)
+       assert.GreaterOrEqual(t, atomic.LoadInt32(&getAsyncCount), 
int32(len(leaves)))
+}

Reply via email to