(hudi) 26/40: [HUDI-7438] Improve the filtering of PRs and pagination in scheduled workflow (#10747)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-0.14.1-spark35-scala213
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 61939468979144e83700085dcb984dca964f6d53
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Sat Feb 24 21:53:39 2024 -0800

    [HUDI-7438] Improve the filtering of PRs and pagination in scheduled 
workflow (#10747)
---
 .github/workflows/azure_ci.js            | 11 +++++++----
 .github/workflows/scheduled_workflow.yml | 16 ++++++++++------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/azure_ci.js b/.github/workflows/azure_ci.js
index 98ba39488b0..737b8db9917 100644
--- a/.github/workflows/azure_ci.js
+++ b/.github/workflows/azure_ci.js
@@ -21,21 +21,24 @@ async function checkAzureCiAndCreateCommitStatus({ github, 
context, prNumber, la
   console.log(`- Checking Azure CI status of PR: ${prNumber} 
${latestCommitHash}`);
   const botUsername = 'hudi-bot';
 
-  const comments = await github.rest.issues.listComments({
+  const comments = await github.paginate(github.rest.issues.listComments, {
     owner: context.repo.owner,
     repo: context.repo.repo,
     issue_number: prNumber,
+    sort: 'updated',
+    direction: 'desc',
+    per_page: 100
   });
 
   // Find the latest comment from hudi-bot containing the Azure CI report
-  const botComments = comments.data.filter(comment => comment.user.login === 
botUsername);
-  const lastComment = botComments.pop();
+  const botComments = comments.filter(comment => comment.user.login === 
botUsername);
 
   let status = 'pending';
   let message = 'In progress';
   let azureRunLink = '';
 
-  if (lastComment) {
+  if (botComments.length > 0) {
+    const lastComment = botComments[0];
     const reportPrefix = `${latestCommitHash} Azure: `
     const successReportString = `${reportPrefix}[SUCCESS]`
     const failureReportString = `${reportPrefix}[FAILURE]`
diff --git a/.github/workflows/scheduled_workflow.yml 
b/.github/workflows/scheduled_workflow.yml
index 39d291fed40..4e17ee12990 100644
--- a/.github/workflows/scheduled_workflow.yml
+++ b/.github/workflows/scheduled_workflow.yml
@@ -44,16 +44,20 @@ jobs:
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const since = new Date(new Date().getTime() - (330 * 
1000)).toISOString();
-            const query = `repo:${context.repo.owner}/${context.repo.repo} 
type:pr updated:>=${since}`;
-            const response = await github.rest.search.issuesAndPullRequests({
-              q: query
+            // Cron schedule may not be reliable so giving buffer time to 
avoid missing recent PRs
+            const since = new Date(new Date().getTime() - (900 * 
1000)).toISOString();
+            const query = `repo:${context.repo.owner}/${context.repo.repo} 
type:pr state:open base:master updated:>=${since}`;
+            const openPrs = await 
github.paginate(github.rest.search.issuesAndPullRequests, {
+              q: query,
+              sort: 'updated',
+              order: 'desc',
+              per_page: 100
             });
             
-            // Filter for open PRs
-            const openPrs = response.data.items.filter(pr => pr.state === 
'open');
             const checkAzureCiAndCreateCommitStatus = 
require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`);
             
+            console.log(`Number of PRs to process: ${openPrs.length}`);
+            
             for (const pr of openPrs) {
               console.log(`*** Processing PR: ${pr.title}, URL: 
${pr.html_url}`);