github-actions[bot] commented on code in PR #65541: URL: https://github.com/apache/doris/pull/65541#discussion_r3569680587
########## regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_py_udf_complex_zip.py: ########## @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build the Python complex-UDF archive with its jieba dependency.""" + +import argparse +import hashlib +from pathlib import Path, PurePosixPath +import tarfile +import zipfile + + +JIEBA_VERSION = "0.42.1" +JIEBA_ARCHIVE_SHA256 = "055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2" +UDF_SOURCES = ( + "business_logic.py", + "complex_udaf.py", + "complex_udtf.py", + "external_api.py", + "nlp_chinese.py", +) +REQUIRED_JIEBA_FILES = ( + "jieba/__init__.py", + "jieba/dict.txt", + "jieba/analyse/idf.txt", + "jieba/posseg/__init__.py", +) +ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) + + +def parse_args(): + script_dir = Path(__file__).resolve().parent + parser = argparse.ArgumentParser( + description="Build py_udf_complex.zip from a verified jieba source archive" + ) + parser.add_argument( + "--jieba-archive", + required=True, + type=Path, + help="path to the jieba-0.42.1 source tar.gz", + ) + parser.add_argument( + "--output", + default=script_dir / "py_udf_complex.zip", + type=Path, + help="output zip path", + ) + return parser.parse_args() + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_entries(script_dir, jieba_archive): + actual_sha256 = sha256(jieba_archive) + if actual_sha256 != JIEBA_ARCHIVE_SHA256: + raise ValueError( + "unexpected jieba archive SHA-256: " + f"expected {JIEBA_ARCHIVE_SHA256}, got {actual_sha256}" + ) + + entries = {name: (script_dir / name).read_bytes() for name in UDF_SOURCES} + entries["THIRD_PARTY_LICENSES/jieba.txt"] = ( + script_dir / "jieba.LICENSE" + ).read_bytes() + + archive_prefix = PurePosixPath(f"jieba-{JIEBA_VERSION}") / "jieba" + with tarfile.open(jieba_archive, "r:gz") as archive: Review Comment: This makes the existing shared complex-UDF archive much heavier for suites that do not use `jieba`. The old zip at the PR base is only 18 KB compressed / 73 KB uncompressed, while this one is 5.25 MB compressed / 18.6 MB uncompressed because of `jieba/dict.txt`, `jieba/analyse/idf.txt`, and the POS/finalseg tables. That would be fine if only the NLP suite loaded it, but `test_python_udf_business_logic`, `test_python_udf_external_api`, `test_python_udaf_complex`, and `test_python_udtf_complex` all still point at the same `py_udf_complex.zip`. On BE this is paid per function id, not once per identical zip: scalar UDF, UDAF, and UDTF open paths pass `_fn.id`/`_t_fn.id` into `UserFunctionCache::get_pypath`, `_get_cache_entry` keys `_entry_map` by `fid`, `_make_lib_file` includes that id in the cached filename, and `_load_cache_entry` unzips each `PY_ZIP`. So the 45 non-NLP functions in these suites now copy/extract the 18.6 MB `jieba` payload even though none of them imports it, which is hundreds of MB of avoidable per-BE IO/disk churn and leaves much larger cache directories if a suite aborts before cleanup. Please split out an NLP-specific archive (for example `nlp_chinese.py` + `jieba` + its license) and have only `test_python_udf_nlp_chinese.groovy` use that, leaving the common archive small for the other complex UDF cases. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
