This is an automated email from the ASF dual-hosted git repository. jscheffl pushed a commit to branch de-dup-experiment in repository https://gitbox.apache.org/repos/asf/airflow-site.git
commit 2f92330ffc166f542e279232bb42ca7145d7595b Author: Jens Scheffler <[email protected]> AuthorDate: Sun Apr 27 10:02:35 2025 +0200 de-dup experiment --- de-dup-files.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/de-dup-files.py b/de-dup-files.py new file mode 100755 index 0000000000..33fa471e7d --- /dev/null +++ b/de-dup-files.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +import hashlib +from pathlib import Path +import re + + +BUF_SIZE = 65536 + +Path("de-dup-files").mkdir(exist_ok=True) +extension_map: dict[str, int] = {} +hash_map: dict[str, int] = {} +html_hash_map: dict[str, int] = {} +for file in Path("docs-archive").rglob("*"): + if file.is_file(): + suffix = file.suffix + file_size = file.stat().st_size + extension_map[suffix] = extension_map.get(suffix, 0) + file_size + + sha1 = hashlib.sha1() + if suffix == ".html": + html_content = file.read_text("utf-8") + html_content = re.sub( + r"<span class=\"bodytext__medium--greyish-brown\">Version: </span><span class=\"version\">[0-9\.]+</span>", + r"<span class=\"bodytext__medium--greyish-brown\">Version: </span><span class=\"version\"><script lang=\"javascript\">document.write(location.href.replace(/.*\/([0-9\.]+)\/.*/, \"$1\"));</script></span>", + html_content, + ) + sha1.update(html_content.encode("utf-8")) + file_sha1 = sha1.hexdigest() + html_hash_map[file_sha1] = file_size + else: + with file.open('rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha1.update(data) + + file_sha1 = sha1.hexdigest() + print("SHA1: {0}".format(file_sha1)) + hash_map[file_sha1] = file_size + +print("Extension map:") +for ext, size in extension_map.items(): + print(f"{ext}: {size / 1024:.2f} KB") + +print(f"File size on hash: {sum(hash_map.values()) / 1024:.2f} KB") +print(f"File size on extension: {sum(extension_map.values()) / 1024:.2f} KB") +print(f"File size on html: {sum(html_hash_map.values()) / 1024:.2f} KB")
