This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch download-image in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 5a77a02f3b00124a3ba1cbcf03e75f36ad451bf0 Author: Xuanwo <[email protected]> AuthorDate: Mon Mar 10 18:59:49 2025 +0800 website: Download all images to local and rewrite the url in html Signed-off-by: Xuanwo <[email protected]> --- website/docusaurus.config.js | 2 + website/package.json | 1 + website/plugins/image-ssr-plugin.js | 127 +++++++++++++++++++++++++++++++++++ website/pnpm-lock.yaml | 128 ++++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 7de64b4e1..b8ab4c827 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -140,6 +140,8 @@ const config = { }, ], require.resolve("docusaurus-lunr-search"), + // This plugin will download all images to local and rewrite the url in html. + require.resolve("./plugins/image-ssr-plugin"), ], themeConfig: diff --git a/website/package.json b/website/package.json index 280a5b982..808984a80 100644 --- a/website/package.json +++ b/website/package.json @@ -18,6 +18,7 @@ "@docusaurus/plugin-client-redirects": "^3.4.0", "@docusaurus/preset-classic": "^3.4.0", "@mdx-js/react": "^3.0.1", + "axios": "^1.8.2", "clsx": "^1.2.1", "docusaurus-lunr-search": "^3.4.0", "docusaurus-plugin-image-zoom": "^0.1.4", diff --git a/website/plugins/image-ssr-plugin.js b/website/plugins/image-ssr-plugin.js new file mode 100644 index 000000000..be2c6e855 --- /dev/null +++ b/website/plugins/image-ssr-plugin.js @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const path = require("path"); +const fs = require("fs-extra"); +const axios = require("axios"); +const { createHash } = require("crypto"); +const cheerio = require("cheerio"); + +module.exports = function () { + return { + name: "docusaurus-image-ssr", + + async postBuild({ outDir }) { + console.log("Localizing external images in build output..."); + + const imagesDir = path.join(outDir, "img/external"); + await fs.ensureDir(imagesDir); + + const imageCache = new Map(); + + async function downloadImage(url) { + if (imageCache.has(url)) { + return imageCache.get(url); + } + + try { + const hash = createHash("md5").update(url).digest("hex"); + const ext = path.extname(url) || ".jpg"; + const filename = `${hash}${ext}`; + const imagePath = path.join(imagesDir, filename); + + if (!fs.existsSync(imagePath)) { + console.log(`Downloading: ${url}`); + const response = await axios({ + url: url, + responseType: "arraybuffer", + timeout: 15000, + headers: { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + }, + }); + + await fs.writeFile(imagePath, response.data); + } + + const localUrl = `/img/external/${filename}`; + imageCache.set(url, localUrl); + return localUrl; + } catch (error) { + console.error(`Failed to download ${url}: ${error.message}`); + return url; + } + } + + const htmlFiles = []; + + async function findHtmlFiles(dir) { + const files = await fs.readdir(dir); + + for (const file of files) { + const filePath = path.join(dir, file); + const stat = await fs.stat(filePath); + + if (stat.isDirectory()) { + await findHtmlFiles(filePath); + } else if (file.endsWith(".html")) { + htmlFiles.push(filePath); + } + } + } + + await findHtmlFiles(outDir); + console.log(`Found ${htmlFiles.length} HTML files`); + + for (const htmlFile of htmlFiles) { + const html = await fs.readFile(htmlFile, "utf8"); + const $ = cheerio.load(html); + + const externalImages = $('img[src^="http"]'); + if (externalImages.length === 0) continue; + + console.log( + `Processing ${externalImages.length} images in ${htmlFile}`, + ); + + // 替换图片链接 + const promises = []; + + externalImages.each((_, img) => { + const element = $(img); + const url = element.attr("src"); + + promises.push( + downloadImage(url).then((localUrl) => { + element.attr("src", localUrl); + element.attr("data-original-src", url); + }), + ); + }); + + await Promise.all(promises); + + await fs.writeFile(htmlFile, $.html()); + } + + console.log("Image localization complete!"); + }, + }; +}; diff --git a/website/pnpm-lock.yaml b/website/pnpm-lock.yaml index 41dea77a4..0d260b746 100644 --- a/website/pnpm-lock.yaml +++ b/website/pnpm-lock.yaml @@ -17,6 +17,9 @@ dependencies: '@mdx-js/react': specifier: ^3.0.1 version: 3.1.0(@types/[email protected])([email protected]) + axios: + specifier: ^1.8.2 + version: 1.8.2 clsx: specifier: ^1.2.1 version: 1.2.1 @@ -3448,6 +3451,10 @@ packages: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} hasBin: true + /[email protected]: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: false + /[email protected]: resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} engines: {node: '>= 4.0.0'} @@ -3475,6 +3482,16 @@ packages: postcss-value-parser: 4.2.0 dev: false + /[email protected]: + resolution: {integrity: sha512-ls4GYBm5aig9vWx8AWDSGLpnpDQRtWAfrjU+EuytuODrFBkqesN2RkOQCBzrA1RQNHw1SmRMSDDDSwzNAYQ6Rg==} + dependencies: + follow-redirects: 1.15.9 + form-data: 4.0.2 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + /[email protected](@babel/[email protected])([email protected]): resolution: {integrity: sha512-fqe8naHt46e0yIdkjUZYqddSXfej3AHajX+CSO5X7oy0EmPc6o5Xh+RClNoHjnieWz9AW4kZxW9yyFMhVB1QLA==} engines: {node: '>= 14.15.0'} @@ -3672,6 +3689,14 @@ packages: responselike: 3.0.0 dev: false + /[email protected]: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + dev: false + /[email protected]: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} @@ -3874,6 +3899,13 @@ packages: engines: {node: '>=10'} dev: false + /[email protected]: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: false + /[email protected]: resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} dev: false @@ -4378,6 +4410,11 @@ packages: slash: 3.0.0 dev: false + /[email protected]: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: false + /[email protected]: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} engines: {node: '>= 0.6'} @@ -4550,6 +4587,15 @@ packages: is-obj: 2.0.0 dev: false + /[email protected]: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + dev: false + /[email protected]: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: false @@ -4625,6 +4671,11 @@ packages: get-intrinsic: 1.2.4 dev: false + /[email protected]: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + dev: false + /[email protected]: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} @@ -4633,6 +4684,23 @@ packages: /[email protected]: resolution: {integrity: sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==} + /[email protected]: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + dependencies: + es-errors: 1.3.0 + dev: false + + /[email protected]: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + dev: false + /[email protected]: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} dependencies: @@ -5018,6 +5086,16 @@ packages: engines: {node: '>= 14.17'} dev: false + /[email protected]: + resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + mime-types: 2.1.35 + dev: false + /[email protected]: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} @@ -5108,10 +5186,34 @@ packages: hasown: 2.0.2 dev: false + /[email protected]: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + dev: false + /[email protected]: resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} dev: false + /[email protected]: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + dev: false + /[email protected]: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} @@ -5207,6 +5309,11 @@ packages: get-intrinsic: 1.2.4 dev: false + /[email protected]: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + dev: false + /[email protected]: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} engines: {node: '>=14.16'} @@ -5272,6 +5379,18 @@ packages: engines: {node: '>= 0.4'} dev: false + /[email protected]: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + dev: false + + /[email protected]: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + dependencies: + has-symbols: 1.0.3 + dev: false + /[email protected]: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} dev: false @@ -6202,6 +6321,11 @@ packages: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} dev: false + /[email protected]: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + dev: false + /[email protected]: resolution: {integrity: sha512-JUpYOqKI4mM3sZcNxmF/ox04XYFFkNwr0CFlrQIkCwbvH0xzMCqkMqAde9wRd80VAhaUrwFwKm2nxretdT1h7Q==} dependencies: @@ -7792,6 +7916,10 @@ packages: ipaddr.js: 1.9.1 dev: false + /[email protected]: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: false + /[email protected]: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'}
