This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory-site.git
The following commit(s) were added to refs/heads/main by this push:
new 3360d604e fix: add retry logic and rate limiting to avatar download
script (#379)
3360d604e is described below
commit 3360d604ecf770cd403182a6ee3446b708c86630
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Dec 26 15:20:21 2025 +0800
fix: add retry logic and rate limiting to avatar download script (#379)
## Summary
- Add retry logic with exponential backoff (3 retries, starting at 1s
delay) to handle transient network errors
- Add batch processing to limit concurrent requests to 5 at a time
- Move file existence check before network request for efficiency
## Context
CI was failing on PRs with transient network errors like:
```
TypeError: fetch failed
cause: SocketError: other side closed
```
This happened because the script was downloading all avatars in parallel
with no retry mechanism. The changes make the script more resilient to
network issues.
## Test plan
- [x] Script syntax verified
- [ ] CI passes on this PR
---
scripts/download_avatars/script.js | 41 +++++++++++++++++++++++++++++++-------
1 file changed, 34 insertions(+), 7 deletions(-)
diff --git a/scripts/download_avatars/script.js
b/scripts/download_avatars/script.js
index 47946c4dd..2ec4fa35d 100644
--- a/scripts/download_avatars/script.js
+++ b/scripts/download_avatars/script.js
@@ -15,27 +15,54 @@ async function ensureDir() {
await fs.mkdir(avatarDir);
}
+async function sleep(ms) {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function fetchWithRetry(url, retries = 3, delay = 1000) {
+ for (let i = 0; i < retries; i++) {
+ try {
+ const response = await fetch(url);
+ if (!response.ok) {
+ throw new Error(`HTTP error: ${response.status}`);
+ }
+ return response;
+ } catch (error) {
+ if (i === retries - 1) {
+ throw error;
+ }
+ console.log(`Retry ${i + 1}/${retries} for ${url}:
${error.message}`);
+ await sleep(delay * Math.pow(2, i));
+ }
+ }
+}
+
async function download(name, link) {
- const response = await fetch(link);
- const image = Buffer.from(await response.arrayBuffer());
const localFile = path.join(avatarDir, `${name}.png`);
if (fileExists(localFile)) {
return;
}
+ const response = await fetchWithRetry(link);
+ const image = Buffer.from(await response.arrayBuffer());
await fs.writeFile(localFile, image);
}
+async function processBatch(items, batchSize, fn) {
+ for (let i = 0; i < items.length; i += batchSize) {
+ const batch = items.slice(i, i + batchSize);
+ await Promise.all(batch.map(fn));
+ }
+}
+
async function exec() {
await ensureDir();
const fileDir = path.join(__dirname, "../../", "blog", "authors.yml");
const fileContent = (await fs.readFile(fileDir)).toString();
const authors = parse(fileContent);
- await Promise.all(Object.entries(authors).map(async ([name, author]) => {
- if (!author?.url) {
- return;
- }
+ const entries = Object.entries(authors).filter(([, author]) =>
author?.url);
+ await processBatch(entries, 5, async ([name, author]) => {
await download(name, `${author.url}.png`);
- }));
+ });
}
exec();
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
