jianguotian commented on code in PR #79:
URL: https://github.com/apache/paimon-mosaic/pull/79#discussion_r3844204995
##########
tools/update_branch_version.sh:
##########
@@ -17,55 +17,110 @@
# limitations under the License.
#
-##
-## Variables with defaults (if not overwritten by environment)
-##
-MVN=${MVN:-mvn}
-
# fail immediately
set -o errexit
set -o nounset
+set -o pipefail
# print command before executing
set -o xtrace
-CURR_DIR=`pwd`
-if [[ `basename $CURR_DIR` != "tools" ]] ; then
+CURR_DIR=$(pwd -P)
+if [[ $(basename "${CURR_DIR}") != "tools" ]] ; then
echo "You have to call the script from the tools/ dir"
exit 1
fi
###########################
-OLD_VERSION=${OLD_VERSION}
-NEW_VERSION=${NEW_VERSION}
-
+OLD_VERSION=${OLD_VERSION:-}
+NEW_VERSION=${NEW_VERSION:-}
-if [ -z "${OLD_VERSION}" ]; then
- echo "OLD_VERSION is unset"
- exit 1
+if [[ -z "${OLD_VERSION}" ]]; then
+ echo "OLD_VERSION is unset" >&2
+ exit 1
fi
-if [ -z "${NEW_VERSION}" ]; then
- echo "NEW_VERSION is unset"
- exit 1
+if [[ -z "${NEW_VERSION}" ]]; then
+ echo "NEW_VERSION is unset" >&2
+ exit 1
fi
cd ..
+if [ -n "$(git status --porcelain --untracked-files=all)" ]; then
+ echo "Version updates must start from a clean Git worktree" >&2
+ git status --short >&2
+ exit 1
+fi
+
# Cargo.toml and pyproject.toml never carry the -SNAPSHOT suffix, so strip it
# from both old and new versions when matching/replacing there.
OLD_VERSION_CLEAN=$(echo "$OLD_VERSION" | sed 's/-SNAPSHOT//')
NEW_VERSION_CLEAN=$(echo "$NEW_VERSION" | sed 's/-SNAPSHOT//')
-#change version in all pom files (match both exact and -SNAPSHOT suffix)
-find . -name 'pom.xml' -type f -exec perl -pi -e
's#<version>'$OLD_VERSION'(-SNAPSHOT)?</version>#<version>'$NEW_VERSION'</version>#'
{} \;
-
-#change version in Cargo.toml files
-find . -name 'Cargo.toml' -not -path '*/target/*' -type f -exec perl -pi -e
's#^version = "'$OLD_VERSION_CLEAN'"#version = "'$NEW_VERSION_CLEAN'"#' {} \;
+# Change the project version in all pom files. This is structural rather than
+# textual: a dependency or plugin element carrying the same version string must
+# not be rewritten along with it.
+find . -name 'pom.xml' -not -path '*/target/*' -type f \
+ -exec python3 tools/bump_pom_version.py "$OLD_VERSION" "$NEW_VERSION" {} +
Review Comment:
Fixed. POM updates now accept both OLD_VERSION and OLD_VERSION-SNAPSHOT,
with regression coverage for the documented transition.
##########
tools/native_binary.py:
##########
@@ -0,0 +1,1724 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Validate native-library format, architecture, structure, and exports."""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+
+
+TARGET_ARCHITECTURE = {
+ "x86_64-unknown-linux-gnu": ("ELF", "x86_64"),
+ "aarch64-unknown-linux-gnu": ("ELF", "aarch64"),
+ "aarch64-apple-darwin": ("Mach-O", "aarch64"),
+ "x86_64-pc-windows-msvc": ("PE", "x86_64"),
+}
+
+MACHINE_ARCHITECTURE = {
+ 62: "x86_64",
+ 183: "aarch64",
+}
+PE_MACHINE_ARCHITECTURE = {
+ 0x8664: "x86_64",
+ 0xAA64: "aarch64",
+}
+MACHO_CPU_ARCHITECTURE = {
+ 0x01000007: "x86_64",
+ 0x0100000C: "aarch64",
+}
+
+# A Mosaic native library exports on the order of a hundred symbols; the bound
+# exists to keep hostile input from driving quadratic hash-membership work.
+MAX_DYNAMIC_SYMBOLS = 100_000
+
+MOSAIC_SYMBOL_FAMILIES = {
+ "JNI": {
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderExportSchema",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderFree",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderNumRowGroups",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderOpen",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderOpenRowGroup",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderRowGroupNumRows",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderRowGroupStatMaxs",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderRowGroupStatMins",
+
"Java_org_apache_paimon_mosaic_NativeLib_nativeReaderRowGroupStatNames",
+
"Java_org_apache_paimon_mosaic_NativeLib_nativeReaderRowGroupStatNullCounts",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeReaderSetProjection",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeRowGroupReaderFree",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeRowGroupReaderNumRows",
+
"Java_org_apache_paimon_mosaic_NativeLib_nativeRowGroupReaderReadColumns",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterClose",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterEstimatedSize",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterFree",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterNumRowGroups",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterOpen",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterRowGroupStatMaxs",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterRowGroupStatMins",
+
"Java_org_apache_paimon_mosaic_NativeLib_nativeWriterRowGroupStatNames",
+
"Java_org_apache_paimon_mosaic_NativeLib_nativeWriterRowGroupStatNullCounts",
+ "Java_org_apache_paimon_mosaic_NativeLib_nativeWriterWriteBatch",
+ },
+ "FFI": {
+ "mosaic_last_error",
+ "mosaic_reader_export_schema",
+ "mosaic_reader_free",
+ "mosaic_reader_num_row_groups",
+ "mosaic_reader_open",
+ "mosaic_reader_open_row_group",
+ "mosaic_reader_row_group_num_rows",
+ "mosaic_reader_row_group_num_stats",
+ "mosaic_reader_row_group_stats",
+ "mosaic_reader_set_projection",
+ "mosaic_record_batch_export",
+ "mosaic_record_batch_free",
+ "mosaic_record_batch_num_columns",
+ "mosaic_record_batch_num_rows",
+ "mosaic_row_group_reader_free",
+ "mosaic_row_group_reader_num_rows",
+ "mosaic_row_group_reader_read_columns",
+ "mosaic_writer_close",
+ "mosaic_writer_estimated_file_size",
+ "mosaic_writer_free",
+ "mosaic_writer_num_row_groups",
+ "mosaic_writer_open",
+ "mosaic_writer_options_default",
+ "mosaic_writer_row_group_num_stats",
+ "mosaic_writer_row_group_stats",
+ "mosaic_writer_write_batch",
+ },
+}
+
+
+@dataclass(frozen=True)
+class NativeBinary:
+ binary_format: str
+ architectures: frozenset[str]
+ exported_symbols: frozenset[str]
+
+
+@dataclass(frozen=True)
+class ElfSection:
+ section_type: int
+ flags: int
+ address: int
+ offset: int
+ size: int
+ link: int
+ entry_size: int
+
+
+def require_range(data: bytes, offset: int, size: int, description: str) ->
None:
+ if (
+ offset < 0
+ or size < 0
+ or offset > len(data)
+ or size > len(data) - offset
+ ):
+ raise ValueError(f"{description} is out of bounds")
+
+
+def is_power_of_two(value: int) -> bool:
+ return value > 0 and value & (value - 1) == 0
+
+
+def c_string_bytes(
+ data: bytes, offset: int, limit: int, description: str
+) -> bytes:
+ if offset < 0 or offset >= limit or limit > len(data):
+ raise ValueError(f"{description} is out of bounds")
+ terminator = data.find(b"\0", offset, limit)
+ if terminator < 0:
+ raise ValueError(f"{description} is not null-terminated")
+ return data[offset:terminator]
+
+
+def ascii_symbol(raw_name: bytes) -> str | None:
+ try:
+ return raw_name.decode("ascii")
+ except UnicodeDecodeError:
+ return None
+
+
+def elf_virtual_range(
+ data: bytes,
+ address: int,
+ size: int,
+ load_segments: list[tuple[int, int, int, int]],
+ description: str,
+) -> int:
+ mapped_offsets = set()
+ memory_mapped = False
+ for file_offset, virtual_address, file_size, memory_size in load_segments:
+ if address < virtual_address:
+ continue
+ delta = address - virtual_address
+ if delta > memory_size or size > memory_size - delta:
+ continue
+ memory_mapped = True
+ if delta <= file_size and size <= file_size - delta:
+ mapped_offsets.add(file_offset + delta)
+
+ if not mapped_offsets:
+ if memory_mapped:
+ raise ValueError(f"{description} is not file-backed")
+ raise ValueError(f"{description} is not mapped by an ELF PT_LOAD
segment")
+ if len(mapped_offsets) != 1:
+ raise ValueError(f"{description} has ambiguous ELF PT_LOAD mappings")
+
+ offset = mapped_offsets.pop()
+ require_range(data, offset, size, description)
+ return offset
+
+
+def elf_loader_section(
+ data: bytes,
+ sections: list[ElfSection],
+ load_segments: list[tuple[int, int, int, int]],
+ address: int,
+ section_type: int,
+ minimum_size: int,
+ linked_section: int | None,
+ description: str,
+ section_name: str,
+) -> tuple[int, ElfSection]:
+ offset = elf_virtual_range(
+ data, address, minimum_size, load_segments, description
+ )
+ matches = [
+ (index, section)
+ for index, section in enumerate(sections)
+ if section.section_type == section_type
+ and section.address == address
+ and section.offset == offset
+ ]
+ if not matches:
+ raise ValueError(
+ f"{description} does not reference an {section_name} section"
+ )
+ if len(matches) != 1:
+ raise ValueError(f"{description} references multiple {section_name}
sections")
+
+ index, section = matches[0]
+ if not section.flags & 0x2:
+ raise ValueError(f"{description} section is not allocated")
+ if section.size < minimum_size:
+ raise ValueError(f"{description} is truncated")
+ if linked_section is not None and section.link != linked_section:
+ raise ValueError(f"{description} section does not link to DT_SYMTAB")
+ if (
+ elf_virtual_range(
+ data, address, section.size, load_segments, description
+ )
+ != section.offset
+ ):
+ raise ValueError(f"{description} has inconsistent file mapping")
+ return index, section
+
+
+def elf_sysv_hash(name: bytes) -> int:
+ value = 0
+ for byte in name:
+ value = (value << 4) + byte
+ high = value & 0xF0000000
+ if high:
+ value ^= high >> 24
+ value &= ~high
+ return value & 0xFFFFFFFF
+
+
+def elf_gnu_hash(name: bytes) -> int:
+ value = 5381
+ for byte in name:
+ value = (value * 33 + byte) & 0xFFFFFFFF
+ return value
+
+
+@dataclass(frozen=True)
+class ElfSysvHash:
+ buckets: tuple[int, ...]
+ chains: tuple[int, ...]
+
+ @property
+ def symbol_count(self) -> int:
+ return len(self.chains)
+
+ def contains(self, symbol_index: int, name: bytes) -> bool:
+ index = self.buckets[elf_sysv_hash(name) % len(self.buckets)]
+ while index:
+ if index == symbol_index:
+ return True
+ index = self.chains[index]
+ return False
+
+
+@dataclass(frozen=True)
+class ElfGnuHash:
+ symbol_offset: int
+ bloom_shift: int
+ bloom: tuple[int, ...]
+ buckets: tuple[int, ...]
+ chains: tuple[int, ...]
+ symbol_count: int
+
+ def contains(self, symbol_index: int, name: bytes) -> bool:
+ name_hash = elf_gnu_hash(name)
+ bloom_word = self.bloom[(name_hash // 64) % len(self.bloom)]
+ bloom_mask = (1 << (name_hash % 64)) | (
+ 1 << ((name_hash >> self.bloom_shift) % 64)
+ )
+ if bloom_word & bloom_mask != bloom_mask:
+ return False
+
+ index = self.buckets[name_hash % len(self.buckets)]
+ if (
+ index == 0
+ or symbol_index < index
+ or symbol_index < self.symbol_offset
+ ):
+ return False
+ while True:
+ chain_index = index - self.symbol_offset
+ if chain_index >= len(self.chains):
+ return False
+ chain_hash = self.chains[chain_index]
+ if index == symbol_index:
+ return (chain_hash | 1) == (name_hash | 1)
+ if chain_hash & 1:
+ return False
+ index += 1
+
+
+def parse_elf_sysv_hash(data: bytes, section: ElfSection) -> ElfSysvHash:
+ bucket_count, symbol_count = struct.unpack_from("<II", data,
section.offset)
+ if bucket_count == 0 or symbol_count == 0:
+ raise ValueError("ELF DT_HASH has invalid bucket or symbol count")
+ if section.size != 8 + (bucket_count + symbol_count) * 4:
+ raise ValueError("ELF DT_HASH has an inconsistent table size")
+
+ buckets_offset = section.offset + 8
+ chains_offset = buckets_offset + bucket_count * 4
+ buckets = struct.unpack_from(f"<{bucket_count}I", data, buckets_offset)
+ chains = struct.unpack_from(f"<{symbol_count}I", data, chains_offset)
+ if chains[0] != 0:
+ raise ValueError("ELF DT_HASH chain zero is not a terminator")
+ if any(index >= symbol_count for index in buckets):
+ raise ValueError("ELF DT_HASH bucket index is out of bounds")
+ if any(index >= symbol_count for index in chains):
+ raise ValueError("ELF DT_HASH chain index is out of bounds")
+ # Each dynamic symbol belongs to exactly one bucket's chain. Recording the
+ # owning bucket keeps this linear and distinguishes a cycle within one
chain
+ # from two buckets aliasing the same node, which would let contains()
resolve
+ # a symbol through the wrong bucket.
+ owner: dict[int, int] = {}
+ for bucket_index, bucket in enumerate(buckets):
+ index = bucket
+ while index:
+ previous = owner.get(index)
+ if previous == bucket_index:
+ raise ValueError("ELF DT_HASH contains a chain cycle")
+ if previous is not None:
+ raise ValueError("ELF DT_HASH bucket chains alias")
+ owner[index] = bucket_index
+ index = chains[index]
+ return ElfSysvHash(buckets, chains)
+
+
+def parse_elf_gnu_hash(data: bytes, section: ElfSection) -> ElfGnuHash:
+ (
+ bucket_count,
+ symbol_offset,
+ bloom_count,
+ bloom_shift,
+ ) = struct.unpack_from("<IIII", data, section.offset)
+ if bucket_count == 0 or bloom_count == 0:
+ raise ValueError("ELF DT_GNU_HASH has an invalid header")
+
+ bloom_offset = section.offset + 16
+ buckets_offset = bloom_offset + bloom_count * 8
+ chains_offset = buckets_offset + bucket_count * 4
+ section_end = section.offset + section.size
+ if chains_offset > section_end or (section_end - chains_offset) % 4:
+ raise ValueError("ELF DT_GNU_HASH has an inconsistent table size")
+
+ bloom = struct.unpack_from(f"<{bloom_count}Q", data, bloom_offset)
+ buckets = struct.unpack_from(f"<{bucket_count}I", data, buckets_offset)
+ chain_count = (section_end - chains_offset) // 4
+ chains = struct.unpack_from(f"<{chain_count}I", data, chains_offset)
+ symbol_count = symbol_offset
+ verified = set()
+ for bucket in buckets:
+ if bucket == 0:
+ continue
+ if bucket < symbol_offset:
+ raise ValueError("ELF DT_GNU_HASH bucket precedes the symbol
offset")
+ chain_index = bucket - symbol_offset
+ while chain_index not in verified:
+ if chain_index >= chain_count:
+ raise ValueError("ELF DT_GNU_HASH chain is not terminated")
+ verified.add(chain_index)
+ symbol_count = max(symbol_count, symbol_offset + chain_index + 1)
+ if chains[chain_index] & 1:
+ break
+ chain_index += 1
+ return ElfGnuHash(
+ symbol_offset,
+ bloom_shift,
+ bloom,
+ buckets,
+ chains,
+ symbol_count,
+ )
+
+
+def parse_elf(data: bytes) -> NativeBinary | None:
+ if not data.startswith(b"\x7fELF"):
+ return None
+ if len(data) < 64:
+ raise ValueError("truncated ELF header")
+ if data[4] != 2:
+ raise ValueError(f"unsupported ELF class {data[4]}")
+ if data[5] != 1:
+ raise ValueError(f"unsupported ELF byte order {data[5]}")
+ if data[6] != 1:
+ raise ValueError(f"unsupported ELF identification version {data[6]}")
+
+ (
+ file_type,
+ machine,
+ version,
+ _entry,
+ program_offset,
+ section_offset,
+ _flags,
+ header_size,
+ program_entry_size,
+ program_count,
+ section_entry_size,
+ section_count,
+ section_names_index,
+ ) = struct.unpack_from("<HHIQQQIHHHHHH", data, 16)
+
+ if file_type != 3:
+ raise ValueError(f"ELF file type {file_type} is not ET_DYN")
+ architecture = MACHINE_ARCHITECTURE.get(machine)
+ if architecture is None:
+ raise ValueError(f"unsupported ELF machine {machine}")
+ if version != 1:
+ raise ValueError(f"unsupported ELF version {version}")
+ if header_size != 64:
+ raise ValueError(f"invalid ELF header size {header_size}")
+ if program_count in (0, 0xFFFF):
+ raise ValueError(f"invalid ELF program header count {program_count}")
+ if program_entry_size != 56:
+ raise ValueError(
+ f"invalid ELF program header entry size {program_entry_size}"
+ )
+ if program_offset < header_size:
+ raise ValueError("ELF program header table overlaps the file header")
+ require_range(
+ data,
+ program_offset,
+ program_count * program_entry_size,
+ "ELF program header table",
+ )
+
+ has_load_segment = False
+ load_segments = []
+ executable_load_segments = []
+ dynamic_segments = []
+ for index in range(program_count):
+ offset = program_offset + index * program_entry_size
+ (
+ segment_type,
+ segment_flags,
+ file_offset,
+ virtual_address,
+ _physical_address,
+ file_size,
+ memory_size,
+ alignment,
+ ) = struct.unpack_from("<IIQQQQQQ", data, offset)
+ if alignment not in (0, 1) and not is_power_of_two(alignment):
+ raise ValueError(
+ f"ELF program header {index} has invalid alignment {alignment}"
+ )
+ if file_size:
+ require_range(
+ data,
+ file_offset,
+ file_size,
+ f"ELF program header {index} contents",
+ )
+ if segment_type == 1:
+ if file_size > memory_size:
+ raise ValueError(
+ f"ELF load segment {index} is larger on disk than in
memory"
+ )
+ if (
+ alignment not in (0, 1)
+ and virtual_address % alignment != file_offset % alignment
+ ):
+ raise ValueError(
+ f"ELF load segment {index} has inconsistent alignment"
+ )
+ has_load_segment = has_load_segment or file_size > 0
+ load_segments.append(
+ (file_offset, virtual_address, file_size, memory_size)
+ )
+ if segment_flags & 0x1:
+ executable_load_segments.append(
+ (file_offset, virtual_address, file_size, memory_size)
+ )
+ elif segment_type == 2:
+ if file_size == 0 or file_size % 16:
+ raise ValueError(f"ELF dynamic segment {index} has invalid
size")
+ if file_size > memory_size:
+ raise ValueError(
+ f"ELF dynamic segment {index} is larger on disk than in
memory"
+ )
+ dynamic_segments.append(
+ (index, file_offset, virtual_address, file_size)
+ )
+ elif segment_type == 3:
+ raise ValueError("ELF ET_DYN file contains PT_INTERP and is
executable")
+
+ if not has_load_segment:
+ raise ValueError("ELF shared object has no non-empty PT_LOAD segment")
+ if not dynamic_segments:
+ raise ValueError("ELF shared object has no PT_DYNAMIC segment")
+ if len(dynamic_segments) != 1:
+ raise ValueError("ELF shared object has multiple PT_DYNAMIC segments")
+
+ (
+ dynamic_index,
+ dynamic_offset,
+ dynamic_address,
+ dynamic_size,
+ ) = dynamic_segments[0]
+ mapped_dynamic_offset = elf_virtual_range(
+ data,
+ dynamic_address,
+ dynamic_size,
+ load_segments,
+ f"ELF dynamic segment {dynamic_index}",
+ )
+ if mapped_dynamic_offset != dynamic_offset:
+ raise ValueError(
+ f"ELF dynamic segment {dynamic_index} has inconsistent file
mapping"
+ )
+
+ required_dynamic_tag_names = {
+ 6: "DT_SYMTAB",
+ 5: "DT_STRTAB",
+ 10: "DT_STRSZ",
+ 11: "DT_SYMENT",
+ }
+ hash_dynamic_tag_names = {
+ 4: "DT_HASH",
+ 0x6FFFFEF5: "DT_GNU_HASH",
+ }
+ dynamic_tag_names = {
+ **required_dynamic_tag_names,
+ **hash_dynamic_tag_names,
+ }
+ dynamic_values = {}
+ dynamic_terminated = False
+ for entry_index in range(dynamic_size // 16):
+ tag, value = struct.unpack_from(
+ "<qQ", data, dynamic_offset + entry_index * 16
+ )
+ if tag == 0:
+ dynamic_terminated = True
+ break
+ if tag in dynamic_tag_names:
+ if tag in dynamic_values:
+ raise ValueError(
+ f"ELF PT_DYNAMIC contains duplicate
{dynamic_tag_names[tag]}"
+ )
+ dynamic_values[tag] = value
+ if not dynamic_terminated:
+ raise ValueError("ELF PT_DYNAMIC is not terminated by DT_NULL")
+ for tag, name in required_dynamic_tag_names.items():
+ if tag not in dynamic_values:
+ raise ValueError(f"ELF PT_DYNAMIC is missing {name}")
+ if not any(tag in dynamic_values for tag in hash_dynamic_tag_names):
+ raise ValueError("ELF PT_DYNAMIC is missing DT_HASH or DT_GNU_HASH")
+
+ string_address = dynamic_values[5]
+ symbol_address = dynamic_values[6]
+ string_size = dynamic_values[10]
+ symbol_entry_size = dynamic_values[11]
+ if string_size == 0:
+ raise ValueError("ELF DT_STRSZ is zero")
+ if symbol_entry_size != 24:
+ raise ValueError(f"ELF DT_SYMENT has invalid size {symbol_entry_size}")
+ string_offset = elf_virtual_range(
+ data,
+ string_address,
+ string_size,
+ load_segments,
+ "ELF DT_STRTAB",
+ )
+ symbol_offset = elf_virtual_range(
+ data,
+ symbol_address,
+ symbol_entry_size,
+ load_segments,
+ "ELF DT_SYMTAB",
+ )
+
+ if section_offset == 0:
+ if section_count != 0 or section_names_index != 0:
+ raise ValueError("invalid ELF section header metadata")
+ return NativeBinary("ELF", frozenset({architecture}), frozenset())
+ if section_count in (0, 0xFFFF):
+ raise ValueError(f"invalid ELF section header count {section_count}")
+ if section_names_index == 0xFFFF:
+ raise ValueError("extended ELF section-name indexes are unsupported")
+ if section_entry_size != 64:
+ raise ValueError(
+ f"invalid ELF section header entry size {section_entry_size}"
+ )
+ if section_offset < header_size:
+ raise ValueError("ELF section header table overlaps the file header")
+ require_range(
+ data,
+ section_offset,
+ section_count * section_entry_size,
+ "ELF section header table",
+ )
+ if section_names_index >= section_count:
+ raise ValueError(
+ f"invalid ELF section-name string table index
{section_names_index}"
+ )
+
+ sections = []
+ for index in range(section_count):
+ offset = section_offset + index * section_entry_size
+ (
+ _name,
+ section_type,
+ section_flags,
+ address,
+ file_offset,
+ size,
+ link,
+ _info,
+ alignment,
+ entry_size,
+ ) = struct.unpack_from("<IIQQQQIIQQ", data, offset)
+ if alignment not in (0, 1) and not is_power_of_two(alignment):
+ raise ValueError(
+ f"ELF section header {index} has invalid alignment {alignment}"
+ )
+ if section_type != 8 and size:
+ require_range(
+ data,
+ file_offset,
+ size,
+ f"ELF section header {index} contents",
+ )
+ sections.append(
+ ElfSection(
+ section_type,
+ section_flags,
+ address,
+ file_offset,
+ size,
+ link,
+ entry_size,
+ )
+ )
+
+ matching_symbol_sections = [
+ (index, section)
+ for index, section in enumerate(sections)
+ if section.section_type == 11
+ and section.address == symbol_address
+ and section.offset == symbol_offset
+ ]
+ if not matching_symbol_sections:
+ raise ValueError("ELF DT_SYMTAB does not reference an SHT_DYNSYM
section")
+ if len(matching_symbol_sections) != 1:
+ raise ValueError("ELF DT_SYMTAB references multiple SHT_DYNSYM
sections")
+
+ symbol_section_index, symbol_section = matching_symbol_sections[0]
+ if not symbol_section.flags & 0x2:
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} is not
allocated"
+ )
+ if (
+ symbol_section.entry_size != symbol_entry_size
+ or symbol_section.size < symbol_entry_size
+ or symbol_section.size % symbol_entry_size
+ ):
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} is malformed"
+ )
+ # Membership in the loader hash is checked per symbol, and a table that
+ # funnels every symbol into one bucket makes each check linear. Bound the
+ # entry count so a crafted library cannot force quadratic work.
+ if symbol_section.size // symbol_entry_size > MAX_DYNAMIC_SYMBOLS:
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} declares more "
+ f"than {MAX_DYNAMIC_SYMBOLS} symbols"
+ )
+ if (
+ elf_virtual_range(
+ data,
+ symbol_address,
+ symbol_section.size,
+ load_segments,
+ f"ELF dynamic symbol section {symbol_section_index}",
+ )
+ != symbol_section.offset
+ ):
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} "
+ "has inconsistent file mapping"
+ )
+ loader_hashes = []
+ if 4 in dynamic_values:
+ hash_address = dynamic_values[4]
+ _hash_section_index, hash_section = elf_loader_section(
+ data,
+ sections,
+ load_segments,
+ hash_address,
+ 5,
+ 8,
+ symbol_section_index,
+ "ELF DT_HASH",
+ "SHT_HASH",
+ )
+ sysv_hash = parse_elf_sysv_hash(data, hash_section)
+ if symbol_section.size != sysv_hash.symbol_count * symbol_entry_size:
+ raise ValueError(
+ "ELF DT_HASH symbol count does not match the SHT_DYNSYM size"
+ )
+ loader_hashes.append(sysv_hash)
+ if 0x6FFFFEF5 in dynamic_values:
+ hash_address = dynamic_values[0x6FFFFEF5]
+ _hash_section_index, hash_section = elf_loader_section(
+ data,
+ sections,
+ load_segments,
+ hash_address,
+ 0x6FFFFFF6,
+ 16,
+ symbol_section_index,
+ "ELF DT_GNU_HASH",
+ "SHT_GNU_HASH",
+ )
+ gnu_hash = parse_elf_gnu_hash(data, hash_section)
+ if gnu_hash.symbol_count * symbol_entry_size > symbol_section.size:
+ raise ValueError(
+ "ELF DT_GNU_HASH symbol count exceeds the SHT_DYNSYM size"
+ )
+ loader_hashes.append(gnu_hash)
+ if symbol_section.link >= section_count:
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} "
+ "has invalid string-table link"
+ )
+
+ string_section = sections[symbol_section.link]
+ if string_section.section_type != 3:
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} "
+ "does not link to a string table"
+ )
+ if not string_section.flags & 0x2:
+ raise ValueError(
+ f"ELF dynamic string section {symbol_section.link} is not
allocated"
+ )
+ if (
+ string_section.address != string_address
+ or string_section.offset != string_offset
+ or string_section.size != string_size
+ ):
+ raise ValueError(
+ f"ELF dynamic symbol section {symbol_section_index} "
+ "does not link to DT_STRTAB"
+ )
+ if data[string_offset] != 0:
+ raise ValueError("ELF DT_STRTAB does not start with a null byte")
+ string_limit = string_offset + string_size
+
+ exported_symbols = set()
+ for symbol_index in range(symbol_section.size // symbol_entry_size):
+ entry_offset = symbol_section.offset + symbol_index * symbol_entry_size
+ (
+ name_offset,
+ info,
+ other,
+ symbol_section_index_value,
+ value,
+ symbol_size,
+ ) = struct.unpack_from("<IBBHQQ", data, entry_offset)
+ if name_offset >= string_size:
+ raise ValueError(
+ f"ELF dynamic symbol {symbol_index} has an invalid name offset"
+ )
+ if name_offset == 0:
+ continue
+ binding = info >> 4
+ symbol_type = info & 0x0F
+ visibility = other & 0x03
+ if (
+ binding not in (1, 2)
+ or symbol_type not in (2, 10)
+ or symbol_section_index_value == 0
+ or visibility not in (0, 3)
+ ):
+ continue
+ elf_virtual_range(
+ data,
+ value,
+ max(symbol_size, 1),
+ executable_load_segments,
+ f"ELF dynamic symbol {symbol_index} function",
+ )
+ raw_name = c_string_bytes(
+ data,
+ string_offset + name_offset,
+ string_limit,
+ f"ELF dynamic symbol {symbol_index} name",
+ )
+ name = ascii_symbol(raw_name)
+ if name and all(
+ loader_hash.contains(symbol_index, raw_name)
+ for loader_hash in loader_hashes
+ ):
+ exported_symbols.add(name)
+
+ return NativeBinary(
+ "ELF", frozenset({architecture}), frozenset(exported_symbols)
+ )
+
+
+def pe_rva_span(
+ rva: int,
+ sections: list[tuple[int, int, int, int, int]],
+ headers_size: int,
+ data_size: int,
+ description: str,
+) -> tuple[int, int]:
+ if rva < headers_size:
+ if rva >= data_size:
+ raise ValueError(f"{description} RVA is out of bounds")
+ return rva, min(headers_size, data_size) - rva
+
+ for (
+ virtual_address,
+ virtual_size,
+ file_offset,
+ file_size,
+ _characteristics,
+ ) in sections:
+ mapped_size = max(virtual_size, file_size)
+ if virtual_address <= rva < virtual_address + mapped_size:
+ delta = rva - virtual_address
+ if delta >= file_size:
+ raise ValueError(f"{description} RVA is not file-backed")
+ return file_offset + delta, file_size - delta
+ raise ValueError(f"{description} RVA is not mapped by a PE section")
+
+
+def pe_rva_range(
+ data: bytes,
+ rva: int,
+ size: int,
+ sections: list[tuple[int, int, int, int, int]],
+ headers_size: int,
+ description: str,
+) -> int:
+ offset, available = pe_rva_span(
+ rva, sections, headers_size, len(data), description
+ )
+ if size > available:
+ raise ValueError(f"{description} is out of bounds")
+ require_range(data, offset, size, description)
+ return offset
+
+
+def pe_section_rva_range(
+ data: bytes,
+ rva: int,
+ size: int,
+ sections: list[tuple[int, int, int, int, int]],
+ description: str,
+) -> tuple[int, bool]:
+ for (
+ virtual_address,
+ virtual_size,
+ file_offset,
+ file_size,
+ characteristics,
+ ) in sections:
+ mapped_size = max(virtual_size, file_size)
+ if virtual_address <= rva < virtual_address + mapped_size:
+ delta = rva - virtual_address
+ if delta >= file_size or size > file_size - delta:
+ raise ValueError(f"{description} is not file-backed")
+ offset = file_offset + delta
+ require_range(data, offset, size, description)
+ return offset, bool(characteristics & 0x20000000)
+ raise ValueError(f"{description} is not mapped by a PE section")
+
+
+def parse_pe(data: bytes) -> NativeBinary | None:
+ if not data.startswith(b"MZ"):
+ return None
+ if len(data) < 0x40:
+ raise ValueError("truncated DOS header")
+ pe_offset = struct.unpack_from("<I", data, 0x3C)[0]
+ if pe_offset < 0x40:
+ raise ValueError(f"invalid PE header offset 0x{pe_offset:x}")
+ require_range(data, pe_offset, 24, "PE signature and COFF header")
+ if data[pe_offset : pe_offset + 4] != b"PE\0\0":
+ raise ValueError("invalid PE signature")
+
+ (
+ machine,
+ section_count,
+ _timestamp,
+ _symbol_table_offset,
+ _symbol_count,
+ optional_size,
+ characteristics,
+ ) = struct.unpack_from("<HHIIIHH", data, pe_offset + 4)
+ architecture = PE_MACHINE_ARCHITECTURE.get(machine)
+ if architecture is None:
+ raise ValueError(f"unsupported PE machine 0x{machine:04x}")
+ if section_count == 0:
+ raise ValueError("PE image has no sections")
+ if not characteristics & 0x2000:
+ raise ValueError("PE image does not have the DLL characteristic")
+
+ optional_offset = pe_offset + 24
+ if optional_size < 112:
+ raise ValueError(f"truncated PE optional header ({optional_size}
bytes)")
+ require_range(data, optional_offset, optional_size, "PE optional header")
+ optional_magic = struct.unpack_from("<H", data, optional_offset)[0]
+ if optional_magic != 0x20B:
+ raise ValueError(
+ f"PE optional header magic 0x{optional_magic:04x} is not PE32+"
+ )
+
+ section_alignment = struct.unpack_from("<I", data, optional_offset + 32)[0]
+ file_alignment = struct.unpack_from("<I", data, optional_offset + 36)[0]
+ image_size = struct.unpack_from("<I", data, optional_offset + 56)[0]
+ headers_size = struct.unpack_from("<I", data, optional_offset + 60)[0]
+ directory_count = struct.unpack_from("<I", data, optional_offset + 108)[0]
+ available_directories = (optional_size - 112) // 8
+ if directory_count > available_directories:
+ raise ValueError(
+ "PE optional header does not contain all declared data directories"
+ )
+ if not is_power_of_two(section_alignment):
+ raise ValueError(f"invalid PE section alignment {section_alignment}")
+ if not is_power_of_two(file_alignment):
+ raise ValueError(f"invalid PE file alignment {file_alignment}")
+ if section_alignment < file_alignment:
+ raise ValueError("PE section alignment is smaller than file alignment")
+
+ section_table_offset = optional_offset + optional_size
+ section_table_size = section_count * 40
+ require_range(
+ data, section_table_offset, section_table_size, "PE section table"
+ )
+ section_table_end = section_table_offset + section_table_size
+ if headers_size < section_table_end or headers_size > len(data):
+ raise ValueError(f"invalid PE SizeOfHeaders {headers_size}")
+ if image_size < headers_size:
+ raise ValueError(f"invalid PE SizeOfImage {image_size}")
+
+ sections = []
+ has_file_backed_section = False
+ for index in range(section_count):
+ offset = section_table_offset + index * 40
+ (
+ _name,
+ virtual_size,
+ virtual_address,
+ file_size,
+ file_offset,
+ _relocations,
+ _line_numbers,
+ _relocation_count,
+ _line_number_count,
+ section_characteristics,
+ ) = struct.unpack_from("<8sIIIIIIHHI", data, offset)
+ if virtual_address % section_alignment:
+ raise ValueError(f"PE section {index} has an unaligned virtual
address")
+ if virtual_address + max(virtual_size, file_size) > image_size:
+ raise ValueError(f"PE section {index} exceeds SizeOfImage")
+ if file_size:
+ if file_offset < headers_size or file_offset % file_alignment:
+ raise ValueError(f"PE section {index} has an invalid file
offset")
+ require_range(
+ data, file_offset, file_size, f"PE section {index} contents"
+ )
+ has_file_backed_section = True
+ sections.append(
+ (
+ virtual_address,
+ virtual_size,
+ file_offset,
+ file_size,
+ section_characteristics,
+ )
+ )
+ if not has_file_backed_section:
+ raise ValueError("PE DLL has no file-backed sections")
+
+ exported_symbols = set()
+ if directory_count:
+ export_rva, export_size = struct.unpack_from(
+ "<II", data, optional_offset + 112
+ )
+ if bool(export_rva) != bool(export_size):
+ raise ValueError("PE export directory has an incomplete RVA/size
pair")
+ if export_rva:
+ if export_size < 40:
+ raise ValueError("truncated PE export directory")
+ export_offset = pe_rva_range(
+ data,
+ export_rva,
+ export_size,
+ sections,
+ headers_size,
+ "PE export directory",
+ )
+ (
+ _export_flags,
+ _export_timestamp,
+ _major_version,
+ _minor_version,
+ module_name_rva,
+ _ordinal_base,
+ function_count,
+ name_count,
+ functions_rva,
+ names_rva,
+ ordinals_rva,
+ ) = struct.unpack_from("<IIHHIIIIIII", data, export_offset)
+ if name_count > function_count:
+ raise ValueError(
+ "PE export directory has more names than functions"
+ )
+ if function_count:
+ functions_offset = pe_rva_range(
+ data,
+ functions_rva,
+ function_count * 4,
+ sections,
+ headers_size,
+ "PE export address table",
+ )
+ else:
+ functions_offset = 0
+ if name_count:
+ names_offset = pe_rva_range(
+ data,
+ names_rva,
+ name_count * 4,
+ sections,
+ headers_size,
+ "PE export name table",
+ )
+ ordinals_offset = pe_rva_range(
+ data,
+ ordinals_rva,
+ name_count * 2,
+ sections,
+ headers_size,
+ "PE export ordinal table",
+ )
+ else:
+ names_offset = 0
+ ordinals_offset = 0
+
+ if module_name_rva:
+ module_offset, module_available = pe_rva_span(
+ module_name_rva,
+ sections,
+ headers_size,
+ len(data),
+ "PE export module name",
+ )
+ c_string_bytes(
+ data,
+ module_offset,
+ module_offset + module_available,
+ "PE export module name",
+ )
+
+ previous_name = None
+ for index in range(name_count):
+ ordinal = struct.unpack_from(
+ "<H", data, ordinals_offset + index * 2
+ )[0]
+ if ordinal >= function_count:
+ raise ValueError(
+ f"PE export name {index} has invalid ordinal {ordinal}"
+ )
+ function_rva = struct.unpack_from(
+ "<I", data, functions_offset + ordinal * 4
+ )[0]
+ if function_rva == 0:
+ raise ValueError(
+ f"PE export name {index} points to a null function RVA"
+ )
+ name_rva = struct.unpack_from(
+ "<I", data, names_offset + index * 4
+ )[0]
+ name_offset, name_available = pe_rva_span(
+ name_rva,
+ sections,
+ headers_size,
+ len(data),
+ f"PE export name {index}",
+ )
+ raw_name = c_string_bytes(
+ data,
+ name_offset,
+ name_offset + name_available,
+ f"PE export name {index}",
+ )
+ name = ascii_symbol(raw_name)
+ if name is None:
+ raise ValueError(f"PE export name {index} is not ASCII")
+ if previous_name is not None and raw_name <= previous_name:
+ raise ValueError(
+ "PE export names are not strictly increasing"
+ )
+ previous_name = raw_name
+ if export_rva <= function_rva < export_rva + export_size:
+ forwarder_offset = export_offset + function_rva -
export_rva
+ forwarder = ascii_symbol(
+ c_string_bytes(
+ data,
+ forwarder_offset,
+ export_offset + export_size,
+ f"PE export name {index} forwarder",
+ )
+ )
+ if forwarder is None:
+ raise ValueError(
+ f"PE export name {index} forwarder is not ASCII"
+ )
+ continue
+ _function_offset, executable = pe_section_rva_range(
+ data,
+ function_rva,
+ 1,
+ sections,
+ f"PE export name {index} function RVA",
+ )
+ if executable:
+ exported_symbols.add(name)
+
+ return NativeBinary(
+ "PE", frozenset({architecture}), frozenset(exported_symbols)
+ )
+
+
+def macho_uleb128(
+ data: bytes, offset: int, limit: int, description: str
+) -> tuple[int, int]:
+ value = 0
+ for index in range(10):
+ if offset >= limit:
+ raise ValueError(f"{description} ULEB128 is truncated")
+ byte = data[offset]
+ offset += 1
+ if index == 9 and byte > 1:
+ raise ValueError(f"{description} ULEB128 overflows 64 bits")
+ value |= (byte & 0x7F) << (index * 7)
+ if not byte & 0x80:
+ return value, offset
+ raise ValueError(f"{description} ULEB128 overflows 64 bits")
+
+
+def parse_macho_export_trie(
+ data: bytes, trie_offset: int, trie_size: int
+) -> frozenset[str]:
+ require_range(data, trie_offset, trie_size, "Mach-O export trie")
+ if trie_size == 0:
+ return frozenset()
+
+ trie_end = trie_offset + trie_size
+ exported_symbols = set()
+ active_nodes = set()
+ visited_nodes = set()
+ stack = [(False, 0, b"")]
+ while stack:
+ leaving, node_offset, prefix = stack.pop()
+ if leaving:
+ active_nodes.remove(node_offset)
+ continue
+ if node_offset in active_nodes:
+ raise ValueError("Mach-O export trie contains a cycle")
+ if node_offset in visited_nodes:
+ raise ValueError(
+ "Mach-O export trie references a node more than once"
+ )
+ if node_offset >= trie_size:
+ raise ValueError(
+ "Mach-O export trie child offset is out of bounds"
+ )
+ active_nodes.add(node_offset)
+ visited_nodes.add(node_offset)
+ stack.append((True, node_offset, b""))
+
+ cursor = trie_offset + node_offset
+ terminal_size, cursor = macho_uleb128(
+ data,
+ cursor,
+ trie_end,
+ f"Mach-O export trie node {node_offset} terminal size",
+ )
+ if terminal_size > trie_end - cursor:
+ raise ValueError("Mach-O export trie terminal is out of bounds")
+ terminal_end = cursor + terminal_size
+ if terminal_size:
+ flags, cursor = macho_uleb128(
+ data,
+ cursor,
+ terminal_end,
+ "Mach-O export trie terminal flags",
+ )
+ if flags & 0x03 == 0x03:
+ raise ValueError(
+ "Mach-O export trie terminal has an invalid export kind"
+ )
+ if flags & 0x08 and flags & 0x10:
+ raise ValueError(
+ "Mach-O export trie terminal combines re-export "
+ "and stub/resolver flags"
+ )
+ if flags & 0x08:
+ _ordinal, cursor = macho_uleb128(
+ data,
+ cursor,
+ terminal_end,
+ "Mach-O export trie re-export ordinal",
+ )
+ import_name = c_string_bytes(
+ data,
+ cursor,
+ terminal_end,
+ "Mach-O export trie re-export name",
+ )
+ cursor += len(import_name) + 1
+ else:
+ _address, cursor = macho_uleb128(
+ data,
+ cursor,
+ terminal_end,
+ "Mach-O export trie terminal address",
+ )
+ if flags & 0x10:
+ _resolver, cursor = macho_uleb128(
+ data,
+ cursor,
+ terminal_end,
+ "Mach-O export trie resolver address",
+ )
+ if cursor != terminal_end:
+ raise ValueError(
+ "Mach-O export trie terminal has trailing payload data"
+ )
+ name = ascii_symbol(prefix)
+ if name:
+ exported_symbols.add(name)
Review Comment:
Fixed. Mach-O verification now counts only callable exports mapped to
executable sections.
##########
tools/verify_python_wheels.py:
##########
@@ -0,0 +1,547 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verify native wheels and their artifact-exact legal metadata."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import csv
+import email
+import hashlib
+import hmac
+import io
+import posixpath
+import re
+import stat
+import sys
+from pathlib import Path
+from zipfile import BadZipFile, ZipFile
+
+from native_binary import TARGET_ARCHITECTURE, verify_native_target
+
+
+MAX_ARCHIVE_ENTRY_SIZE = 256 * 1024 * 1024
+MAX_ARCHIVE_TOTAL_SIZE = 1024 * 1024 * 1024
+MAX_ARCHIVE_ENTRIES = 65536
+NATIVE_LIBRARY = {
+ "x86_64-unknown-linux-gnu": "mosaic/libpaimon_mosaic_ffi.so",
+ "aarch64-unknown-linux-gnu": "mosaic/libpaimon_mosaic_ffi.so",
+ "aarch64-apple-darwin": "mosaic/libpaimon_mosaic_ffi.dylib",
+ "x86_64-pc-windows-msvc": "mosaic/paimon_mosaic_ffi.dll",
+}
+
+EXPECTED_WHEEL_TAG = {
+ "x86_64-unknown-linux-gnu": "py3-none-manylinux_2_28_x86_64",
+ "aarch64-unknown-linux-gnu": "py3-none-manylinux_2_28_aarch64",
+ "aarch64-apple-darwin": "py3-none-macosx_11_0_arm64",
+ "x86_64-pc-windows-msvc": "py3-none-win_amd64",
+}
+
+NESTED_LICENSE_MARKERS = (
+ "For Zstandard software",
+ "Apache Arrow",
+)
+
+WINDOWS_ABSOLUTE_PATH = re.compile(r"^[A-Za-z]:/")
+MACHO_MAGICS = {
+ b"\xfe\xed\xfa\xce",
+ b"\xce\xfa\xed\xfe",
+ b"\xfe\xed\xfa\xcf",
+ b"\xcf\xfa\xed\xfe",
+ b"\xca\xfe\xba\xbe",
+ b"\xbe\xba\xfe\xca",
+ b"\xca\xfe\xba\xbf",
+ b"\xbf\xba\xfe\xca",
+}
+
+
+def _validate_target_matrix() -> None:
+ if not (
+ set(NATIVE_LIBRARY)
+ == set(EXPECTED_WHEEL_TAG)
+ == set(TARGET_ARCHITECTURE)
+ ):
+ raise RuntimeError("Python wheel native target matrices are
inconsistent")
+
+
+_validate_target_matrix()
+
+
+def repository_root() -> Path:
+ return Path(__file__).resolve().parent.parent
+
+
+def expand_tag(python_tag: str, abi_tag: str, platform_tag: str) -> set[str]:
+ components = (python_tag, abi_tag, platform_tag)
+ if any(
+ not component
+ or any(not part for part in component.split("."))
+ or any(character.isspace() for character in component)
+ for component in components
+ ):
+ raise ValueError(f"invalid wheel tag components: {components}")
+ return {
+ f"{python}-{abi}-{platform}".lower()
+ for python in python_tag.split(".")
+ for abi in abi_tag.split(".")
+ for platform in platform_tag.split(".")
+ }
+
+
+def parse_wheel_filename(name: str) -> tuple[str, str, set[str]]:
+ if not name.endswith(".whl"):
+ raise ValueError(f"not a wheel filename: {name}")
+ parts = name[:-4].split("-")
+ if len(parts) == 5:
+ distribution, version, python_tag, abi_tag, platform_tag = parts
+ elif len(parts) == 6:
+ distribution, version, build_tag, python_tag, abi_tag, platform_tag =
parts
+ if not re.fullmatch(r"[0-9][A-Za-z0-9_]*", build_tag):
+ raise ValueError(f"invalid wheel build tag: {build_tag}")
+ else:
+ raise ValueError(f"invalid wheel filename: {name}")
+ if (
+ not distribution
+ or not version
+ or any(character.isspace() for character in distribution + version)
+ ):
+ raise ValueError(f"invalid wheel filename: {name}")
+ return (
+ distribution,
+ version,
+ expand_tag(python_tag, abi_tag, platform_tag),
+ )
+
+
+def parse_wheel_metadata_tags(tags: list[str]) -> set[str]:
+ parsed = set()
+ for tag in tags:
+ parts = tag.split("-")
+ if len(parts) != 3:
+ raise ValueError(f"invalid WHEEL Tag field: {tag!r}")
+ expanded = expand_tag(*parts)
+ if parsed.intersection(expanded):
+ raise ValueError(f"duplicate WHEEL Tag field: {tag!r}")
+ parsed.update(expanded)
+ return parsed
+
+
+def target_from_wheel_tags(tags: set[str]) -> str:
+ if any(tag.rsplit("-", 1)[-1].startswith("musllinux_") for tag in tags):
+ raise ValueError("musllinux wheels do not match the supported GNU
targets")
+ matching_targets = [
+ target
+ for target, expected_tag in EXPECTED_WHEEL_TAG.items()
+ if tags == {expected_tag}
+ ]
+ if len(matching_targets) != 1:
+ raise ValueError(
+ f"unsupported wheel tags: {sorted(tags)}; expected exactly one of "
+ f"{sorted(EXPECTED_WHEEL_TAG.values())}"
+ )
+ return matching_targets[0]
+
+
+def target_from_wheel_name(name: str) -> str:
+ _, _, tags = parse_wheel_filename(name)
+ return target_from_wheel_tags(tags)
+
+
+def normalized_distribution(name: str) -> str:
+ return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def parse_dist_info_name(dist_info: str) -> tuple[str, str]:
+ if "/" in dist_info or not dist_info.endswith(".dist-info"):
+ raise ValueError(f"invalid .dist-info directory: {dist_info}")
+ stem = dist_info[: -len(".dist-info")]
+ try:
+ distribution, version = stem.rsplit("-", 1)
+ except ValueError as error:
+ raise ValueError(f"invalid .dist-info directory: {dist_info}") from
error
+ if not distribution or not version:
+ raise ValueError(f"invalid .dist-info directory: {dist_info}")
+ return distribution, version
+
+
+def validate_archive_paths(archive: ZipFile) -> set[str]:
+ names = set()
+ normalized_names = set()
+ # The per-entry cap does not bound the aggregate, and every entry is
streamed
+ # through hashlib to check RECORD, so bound the total and the entry count
too.
+ total_size = 0
+ entries = archive.infolist()
+ if len(entries) > MAX_ARCHIVE_ENTRIES:
+ raise ValueError(
+ f"wheel declares more than {MAX_ARCHIVE_ENTRIES} entries:
{len(entries)}"
+ )
+ for info in entries:
+ raw_name = info.orig_filename
+ if not raw_name or "\x00" in raw_name or raw_name != info.filename:
+ raise ValueError(f"invalid wheel entry path: {raw_name!r}")
+ if "\\" in raw_name:
+ raise ValueError(f"wheel entry uses a backslash: {raw_name!r}")
+ if raw_name.startswith("/") or WINDOWS_ABSOLUTE_PATH.match(raw_name):
Review Comment:
Fixed. All drive-qualified wheel paths, including C:../..., are now rejected.
##########
tools/tests/test_validate_release_tag.py:
##########
@@ -0,0 +1,383 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+
+TOOLS = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(TOOLS))
+
+import validate_release_tag as validator
+
+
+def run(
+ command: list[str],
+ *,
+ cwd: Path,
+ env: dict[str, str] | None = None,
+) -> str:
+ result = subprocess.run(
+ command,
+ cwd=cwd,
+ env=env,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stderr or result.stdout
+ return result.stdout.strip()
+
+
+def generate_key(tmp_path: Path, identity: str) -> tuple[Path, str, Path]:
+ home = tmp_path / identity.replace(" ", "-")
+ home.mkdir(mode=0o700)
+ env = os.environ.copy()
+ env["GNUPGHOME"] = str(home)
+ run(
+ [
+ "gpg",
+ "--batch",
+ "--pinentry-mode",
+ "loopback",
+ "--passphrase",
+ "",
+ "--quick-generate-key",
+ f"{identity} <{identity.replace(' ', '.')}@example.test>",
+ "ed25519",
+ "sign",
+ "0",
+ ],
+ cwd=tmp_path,
+ env=env,
+ )
+ listing = run(
+ ["gpg", "--batch", "--with-colons", "--list-secret-keys"],
+ cwd=tmp_path,
+ env=env,
+ )
+ fingerprint = next(
+ line.split(":")[9] for line in listing.splitlines() if
line.startswith("fpr:")
+ )
+ keys = tmp_path / f"{identity.replace(' ', '-')}.keys"
+ exported = run(
+ ["gpg", "--batch", "--armor", "--export", fingerprint],
+ cwd=tmp_path,
+ env=env,
+ )
+ keys.write_text(exported + "\n", encoding="utf-8")
+ return home, fingerprint, keys
+
+
+def short_temporary_root(platform_name: str = os.name) -> Path:
+ if platform_name == "nt":
+ return Path(tempfile.gettempdir()).resolve()
+ return Path("/tmp").resolve()
+
+
+# sockaddr_un.sun_path holds 104 bytes on macOS and 108 on Linux; gpg-agent
+# refuses to start when $GNUPGHOME/S.gpg-agent does not fit.
+GPG_AGENT_SOCKET_LIMIT = 104
+
+
[email protected](scope="module")
+def signing_keys():
+ temporary_root = short_temporary_root()
+ with tempfile.TemporaryDirectory(
+ prefix="pm-gpg-", dir=temporary_root
+ ) as directory:
+ root = Path(directory).resolve()
+ trusted = generate_key(root, "Trusted Release")
+ untrusted = generate_key(root, "Untrusted Release")
+ yield trusted, untrusted
+
+
+def test_signing_key_homes_fit_the_gpg_agent_socket_limit(signing_keys):
+ for home, _, _ in signing_keys:
+ assert home == home.resolve()
+ assert home.parent.parent == short_temporary_root()
+ assert len(str(home / "S.gpg-agent")) < GPG_AGENT_SOCKET_LIMIT
+
+
+def
test_short_temporary_root_ignores_a_long_platform_temporary_dir(monkeypatch):
+ # The macOS default (/var/folders/<random>/T) is long enough on its own to
+ # push a nested GNUPGHOME past the socket limit, so the POSIX branch must
not
+ # be derived from tempfile.gettempdir().
+ monkeypatch.setattr(tempfile, "gettempdir", lambda: f"/var/folders/{'n' *
64}/T")
+ nested = (
+ short_temporary_root("posix")
+ / "pm-gpg-abcd1234"
+ / "Trusted-Release"
+ / "S.gpg-agent"
+ )
+ assert len(str(nested)) < GPG_AGENT_SOCKET_LIMIT
+
+
+def repository(tmp_path: Path) -> Path:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ run(["git", "init", "-q"], cwd=repo)
+ run(["git", "config", "user.name", "Release Test"], cwd=repo)
+ run(["git", "config", "user.email", "[email protected]"], cwd=repo)
+ commit(repo, "first")
+ return repo
+
+
+def commit(repo: Path, contents: str) -> str:
+ (repo / "payload").write_text(contents, encoding="utf-8")
+ run(["git", "add", "payload"], cwd=repo)
+ run(["git", "commit", "-q", "-m", contents], cwd=repo)
+ return run(["git", "rev-parse", "HEAD"], cwd=repo)
+
+
+def sign_tag(repo: Path, tag: str, home: Path, fingerprint: str) -> None:
+ env = os.environ.copy()
+ env["GNUPGHOME"] = str(home)
+ run(
+ [
+ "git",
+ "-c",
+ "gpg.program=gpg",
+ "-c",
+ f"user.signingkey={fingerprint}",
+ "tag",
+ "-s",
+ "-m",
+ tag,
+ tag,
+ ],
+ cwd=repo,
+ env=env,
+ )
+
+
[email protected](
+ "tag",
+ [
+ "1.2.3",
+ "v01.2.3",
+ "v1.02.3",
+ "v1.2.03",
+ "v1.2.3-rc0",
+ "v1.2.3-rc01",
+ "v1.2.3-RC1",
+ "v1.2.3-extra",
+ ],
+)
+def test_parse_release_tag_rejects_noncanonical_names(tag):
+ with pytest.raises(validator.TagValidationError, match="not a canonical"):
+ validator.parse_release_tag(tag)
+
+
+def test_signed_rc_and_final_on_same_commit_are_accepted(tmp_path,
signing_keys):
+ (home, fingerprint, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(repo, "v1.2.3-rc1", home, fingerprint)
+ sign_tag(repo, "v1.2.3", home, fingerprint)
+
+ rc = validator.validate_release_tag(repo, "v1.2.3-rc1", keys)
+ final = validator.validate_release_tag(repo, "v1.2.3", keys)
+
+ assert rc.matching_rc is None
+ assert final.commit == rc.commit
+ assert final.matching_rc == "v1.2.3-rc1"
+
+
+def test_main_returns_success_and_failure_status(
+ tmp_path, signing_keys, monkeypatch, capsys
+):
+ (home, fingerprint, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ tag = "v1.2.4-rc1"
+ sign_tag(repo, tag, home, fingerprint)
+ arguments = [
+ "validate_release_tag.py",
+ tag,
+ "--keys-file",
+ str(keys),
+ "--repository",
+ str(repo),
+ ]
+
+ monkeypatch.setattr(sys, "argv", arguments)
+ assert validator.main() == 0
+ capsys.readouterr()
+
+ commit(repo, "after tag")
+ monkeypatch.setattr(sys, "argv", arguments)
+ assert validator.main() == 1
+ captured = capsys.readouterr()
+ assert "release tag validation failed" in captured.err
+
+
+def test_invalid_extra_rc_does_not_hide_a_valid_matching_rc(tmp_path,
signing_keys):
+ (home, fingerprint, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(repo, "v1.3.0-rc1", home, fingerprint)
+ run(["git", "config", "tag.gpgSign", "true"], cwd=repo)
+ env = os.environ.copy()
+ env["GIT_EDITOR"] = "false"
+ run(
+ ["git", "-c", "tag.gpgSign=false", "tag", "v1.3.0-rc2"],
+ cwd=repo,
+ env=env,
+ )
+ sign_tag(repo, "v1.3.0", home, fingerprint)
+
+ final = validator.validate_release_tag(repo, "v1.3.0", keys)
+
+ assert final.matching_rc == "v1.3.0-rc1"
+
+
+def test_final_tag_requires_matching_rc_signed_by_supplied_keys(
+ tmp_path, signing_keys
+):
+ (trusted_home, trusted_fingerprint, trusted_keys), (
+ untrusted_home,
+ untrusted_fingerprint,
+ _,
+ ) = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(
+ repo,
+ "v1.4.0-rc1",
+ untrusted_home,
+ untrusted_fingerprint,
+ )
+ sign_tag(repo, "v1.4.0", trusted_home, trusted_fingerprint)
+
+ with pytest.raises(
+ validator.TagValidationError,
+ match="no matching RC tag.*valid ASF Paimon signature",
+ ):
+ validator.validate_release_tag(
+ repo,
+ "v1.4.0",
+ trusted_keys,
+ )
+
+
+def test_final_tag_rejects_rc_on_a_different_commit(tmp_path, signing_keys):
+ (home, fingerprint, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(repo, "v2.0.0-rc1", home, fingerprint)
+ commit(repo, "final changed")
+ sign_tag(repo, "v2.0.0", home, fingerprint)
+
+ with pytest.raises(validator.TagValidationError, match="same commit"):
+ validator.validate_release_tag(repo, "v2.0.0", keys)
+
+
+def test_tag_validation_rejects_git_replacement_refs(tmp_path, signing_keys):
+ (home, fingerprint, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(repo, "v2.1.0-rc1", home, fingerprint)
+ commit(repo, "replacement")
+ sign_tag(repo, "v2.1.0-rc2", home, fingerprint)
+ first_tag = run(["git", "rev-parse", "refs/tags/v2.1.0-rc1"], cwd=repo)
+ second_tag = run(["git", "rev-parse", "refs/tags/v2.1.0-rc2"], cwd=repo)
+ run(["git", "replace", first_tag, second_tag], cwd=repo)
+
+ with pytest.raises(validator.TagValidationError, match="replacement refs"):
+ validator.validate_release_tag(repo, "v2.1.0-rc1", keys)
+
+
+def test_signature_must_match_a_key_in_supplied_keys(tmp_path, signing_keys):
+ (home, fingerprint, _), (_, _, unrelated_keys) = signing_keys
+ repo = repository(tmp_path)
+ sign_tag(repo, "v3.0.0-rc1", home, fingerprint)
+
+ with pytest.raises(validator.TagValidationError, match="supplied ASF
Paimon KEYS"):
+ validator.validate_release_tag(repo, "v3.0.0-rc1", unrelated_keys)
+
+
+def test_lightweight_tag_is_rejected(tmp_path, signing_keys):
+ (_, _, keys), _ = signing_keys
+ repo = repository(tmp_path)
+ run(["git", "config", "tag.gpgSign", "true"], cwd=repo)
+ env = os.environ.copy()
+ env["GIT_EDITOR"] = "false"
+ run(
+ ["git", "-c", "tag.gpgSign=false", "tag", "v4.0.0-rc1"],
+ cwd=repo,
+ env=env,
+ )
+
+ with pytest.raises(validator.TagValidationError, match="annotated signed"):
+ validator.validate_release_tag(repo, "v4.0.0-rc1", keys)
+
+
+def revoke_key(home: Path, fingerprint: str) -> None:
+ certificate = home / "openpgp-revocs.d" / f"{fingerprint}.rev"
+ # GnuPG prefixes the armor header with a colon so the certificate cannot be
+ # imported by accident.
+ armored = "\n".join(
+ line[1:] if line.startswith(":") else line
+ for line in certificate.read_text(encoding="utf-8").splitlines()
+ )
+ revocation = home.parent / f"{fingerprint}.revoke.asc"
+ revocation.write_text(armored + "\n", encoding="utf-8")
+ env = os.environ.copy()
+ env["GNUPGHOME"] = str(home)
+ run(["gpg", "--batch", "--yes", "--import", str(revocation)], cwd=home,
env=env)
+
+
+def test_tag_signed_by_a_revoked_key_is_rejected(tmp_path):
+ root = Path(tmp_path).resolve()
+ home, fingerprint, keys = generate_key(root, "Compromised Release")
Review Comment:
Fixed. The revoked-key fixture now also uses the short temporary root.
##########
tools/tests/test_verify_release_versions.py:
##########
@@ -0,0 +1,487 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import tempfile
+import tomllib
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from io import StringIO
+from pathlib import Path
+from unittest import mock
+
+
+TOOLS_DIRECTORY = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(TOOLS_DIRECTORY))
+
+import verify_release_versions # noqa: E402
+
+
+PACKAGE_NAMES = {
+ "core": "paimon-mosaic-core",
+ "ffi": "paimon-mosaic-ffi",
+ "jni": "paimon-mosaic-jni",
+ "cli": "paimon-mosaic-cli",
+}
+
+
+class VerifyReleaseVersionsTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.temporary_directory = tempfile.TemporaryDirectory()
+ self.root = Path(self.temporary_directory.name)
+
+ def tearDown(self) -> None:
+ self.temporary_directory.cleanup()
+
+ def write_workspace(
+ self,
+ package_version: str,
+ cli_requirement: str,
+ ffi_requirement: str,
+ jni_requirement: str,
+ ) -> None:
+ (self.root / "Cargo.toml").write_text(
+ "[workspace]\n"
+ 'members = ["core", "ffi", "jni", "cli"]\n'
+ 'resolver = "2"\n',
+ encoding="utf-8",
+ )
+ for directory, package_name in PACKAGE_NAMES.items():
+ package = self.root / directory
+ (package / "src").mkdir(parents=True)
+ (package / "src/lib.rs").write_text("", encoding="utf-8")
+ manifest = (
+ "[package]\n"
+ f'name = "{package_name}"\n'
+ f'version = "{package_version}"\n'
+ 'edition = "2021"\n'
+ )
+ if directory == "cli":
+ manifest += (
+ "\n[dependencies]\n"
+ "# Preserve dependency comments and unrelated
formatting.\n"
+ "paimon-mosaic-core = { "
+ 'path = "../core", '
+ f'version = "{cli_requirement}"'
+ " }\n"
+ )
+ elif directory == "ffi":
+ manifest += (
+ "\n[dependencies.mosaic-core]\n"
+ 'package = "paimon-mosaic-core"\n'
+ 'path = "../core"\n'
+ f'version = "{ffi_requirement}"\n'
+ )
+ elif directory == "jni":
+ manifest += (
+ "\n[target.'cfg(unix)'.build-dependencies]\n"
+ "mosaic-core = { "
+ 'package = "paimon-mosaic-core", '
+ 'path = "../core", '
+ f'version = "{jni_requirement}"'
+ " }\n"
+ )
+ (package / "Cargo.toml").write_text(manifest, encoding="utf-8")
+
+ def write_release_components(
+ self, version: str, java_snapshot: bool = True
+ ) -> None:
+ (self.root / "java").mkdir()
+ java_version = f"{version}-SNAPSHOT" if java_snapshot else version
+ (self.root / "java/pom.xml").write_text(
+ '<project xmlns="http://maven.apache.org/POM/4.0.0";>\n'
+ f" <version>{java_version}</version>\n"
+ "</project>\n",
+ encoding="utf-8",
+ )
+ (self.root / "python").mkdir()
+ (self.root / "python/pyproject.toml").write_text(
+ "[project]\n" f'version = "{version}"\n',
+ encoding="utf-8",
+ )
+ lockfile = ["version = 4", ""]
+ for package_name in PACKAGE_NAMES.values():
+ lockfile.extend(
+ [
+ "[[package]]",
+ f'name = "{package_name}"',
+ f'version = "{version}"',
+ "",
+ ]
+ )
+ (self.root / "Cargo.lock").write_text(
+ "\n".join(lockfile), encoding="utf-8"
+ )
+
+ def load_manifest(self, directory: str) -> dict:
+ with (self.root / directory / "Cargo.toml").open("rb") as file:
+ return tomllib.load(file)
+
+ def test_updates_packages_and_all_versioned_workspace_path_dependencies(
+ self,
+ ) -> None:
+ self.write_workspace(
+ "0.3.0",
+ "0.3.0",
+ "^0.3.0",
+ ">=0.3.0, <0.4.0",
+ )
+
+ updated = verify_release_versions.update_cargo_versions(
+ self.root, "0.3.0", "0.4.0"
+ )
+
+ self.assertEqual(
+ {path.relative_to(self.root).as_posix() for path in updated},
Review Comment:
Fixed. The fixture root is now resolved before relativizing updater results.
##########
tools/verify_java_jars.py:
##########
@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verify main and classifier JAR licensing matches their bundled content."""
+
+from __future__ import annotations
+
+import argparse
+import posixpath
+import stat
+import sys
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from zipfile import BadZipFile, ZipFile, ZipInfo
+
+from native_binary import TARGET_ARCHITECTURE, verify_native_target
+
+
+MAX_ARCHIVE_ENTRY_SIZE = 256 * 1024 * 1024
+MAX_ARCHIVE_TOTAL_SIZE = 1024 * 1024 * 1024
+MAX_ARCHIVE_ENTRIES = 65536
+MAX_JAVA_CLASS_SIZE = 16 * 1024 * 1024
+TARGETS = (
+ "x86_64-unknown-linux-gnu",
+ "aarch64-unknown-linux-gnu",
+ "aarch64-apple-darwin",
+ "x86_64-pc-windows-msvc",
+)
+NATIVE_ENTRIES = {
+ "native/linux/x86_64/libpaimon_mosaic_jni.so": "x86_64-unknown-linux-gnu",
+ "native/linux/aarch64/libpaimon_mosaic_jni.so":
"aarch64-unknown-linux-gnu",
+ "native/macos/aarch64/libpaimon_mosaic_jni.dylib": "aarch64-apple-darwin",
+ "native/windows/x86_64/paimon_mosaic_jni.dll": "x86_64-pc-windows-msvc",
+}
+NATIVE_SUFFIXES = (".so", ".dylib", ".dll")
+JAVA_CLASS_MAGIC = b"\xca\xfe\xba\xbe"
+MACHO_MAGICS = {
+ JAVA_CLASS_MAGIC,
+ b"\xbe\xba\xfe\xca",
+ b"\xca\xfe\xba\xbf",
+ b"\xbf\xba\xfe\xca",
+ b"\xce\xfa\xed\xfe",
+ b"\xcf\xfa\xed\xfe",
+ b"\xfe\xed\xfa\xce",
+ b"\xfe\xed\xfa\xcf",
+}
+NESTED_LICENSE_MARKERS = (
+ "For Zstandard software",
+ "Apache Arrow",
+)
+
+
+def _validate_target_matrix() -> None:
+ if not (
+ set(TARGETS)
+ == set(NATIVE_ENTRIES.values())
+ == set(TARGET_ARCHITECTURE)
+ ):
+ raise RuntimeError("Java native target matrices are inconsistent")
+
+
+_validate_target_matrix()
+
+
+def repository_root() -> Path:
+ return Path(__file__).resolve().parent.parent
+
+
+def validated_entries(archive: ZipFile) -> dict[str, ZipInfo]:
+ entries: dict[str, ZipInfo] = {}
+ normalized_names: dict[str, str] = {}
+ # The per-entry cap does not bound the aggregate, so bound the total and
the
+ # entry count too.
+ total_size = 0
+ infos = archive.infolist()
+ if len(infos) > MAX_ARCHIVE_ENTRIES:
+ raise ValueError(
+ f"archive declares more than {MAX_ARCHIVE_ENTRIES} entries:
{len(infos)}"
+ )
+ for info in infos:
+ name = info.orig_filename
+ if not name or "\x00" in name or name != info.filename:
+ raise ValueError(f"invalid archive entry path: {name!r}")
+ if "\\" in name:
+ raise ValueError(f"archive entry uses a backslash: {name!r}")
+ if PurePosixPath(name).is_absolute() or
PureWindowsPath(name).is_absolute():
+ raise ValueError(f"archive entry uses an absolute path: {name!r}")
+ if ".." in name.split("/"):
+ raise ValueError(f"archive entry uses a '..' path component:
{name!r}")
+ if stat.S_ISLNK(info.external_attr >> 16):
+ raise ValueError(f"archive entry is a symbolic link: {name!r}")
+ if info.file_size > MAX_ARCHIVE_ENTRY_SIZE:
+ raise ValueError(
+ f"archive entry {name!r} exceeds the size limit of "
+ f"{MAX_ARCHIVE_ENTRY_SIZE} bytes: {info.file_size} bytes"
+ )
+ total_size += info.file_size
+ if total_size > MAX_ARCHIVE_TOTAL_SIZE:
+ raise ValueError(
+ f"archive exceeds the total size limit of "
+ f"{MAX_ARCHIVE_TOTAL_SIZE} bytes"
+ )
+ if name in entries:
+ raise ValueError(f"archive contains duplicate raw entry name:
{name!r}")
+
+ normalized_name = posixpath.normpath(name)
+ previous_name = normalized_names.get(normalized_name)
+ if previous_name is not None:
+ raise ValueError(
+ "archive contains duplicate normalized entry names: "
+ f"{previous_name!r} and {name!r}"
+ )
+
+ entries[name] = info
+ normalized_names[normalized_name] = name
+ return entries
+
+
+def read_java_u2(data: bytes, offset: int) -> tuple[int, int] | None:
+ if offset > len(data) - 2:
+ return None
+ return int.from_bytes(data[offset : offset + 2], "big"), offset + 2
+
+
+def skip_java_attributes(
+ data: bytes, offset: int, count: int
+) -> int | None:
+ for _ in range(count):
+ if offset > len(data) - 6:
+ return None
+ length = int.from_bytes(data[offset + 2 : offset + 6], "big")
+ offset += 6
+ if length > len(data) - offset:
+ return None
+ offset += length
+ return offset
+
+
+def skip_java_members(data: bytes, offset: int) -> int | None:
+ result = read_java_u2(data, offset)
+ if result is None:
+ return None
+ count, offset = result
+ for _ in range(count):
+ if offset > len(data) - 8:
+ return None
+ attributes_count = int.from_bytes(data[offset + 6 : offset + 8], "big")
+ offset = skip_java_attributes(data, offset + 8, attributes_count)
+ if offset is None:
+ return None
+ return offset
+
+
+def is_java_class(data: bytes) -> bool:
+ """Distinguish a complete Java class from Mach-O's shared CAFEBABE
magic."""
+ if len(data) < 10 or not data.startswith(JAVA_CLASS_MAGIC):
+ return False
+ major_version = int.from_bytes(data[6:8], "big")
+ constant_pool_count = int.from_bytes(data[8:10], "big")
+ if not 45 <= major_version <= 100 or constant_pool_count == 0:
+ return False
+
+ offset = 10
+ index = 1
+ while index < constant_pool_count:
+ if offset >= len(data):
+ return False
+ tag = data[offset]
+ offset += 1
+ if tag == 1:
+ result = read_java_u2(data, offset)
+ if result is None:
+ return False
+ length, offset = result
+ if length > len(data) - offset:
+ return False
+ offset += length
+ elif tag in (3, 4, 9, 10, 11, 12, 17, 18):
+ offset += 4
+ elif tag in (5, 6):
+ offset += 8
+ index += 1
+ elif tag in (7, 8, 16, 19, 20):
+ offset += 2
+ elif tag == 15:
+ offset += 3
+ else:
+ return False
+ if offset > len(data):
+ return False
+ index += 1
+
+ if offset > len(data) - 8:
+ return False
+ interfaces_count = int.from_bytes(data[offset + 6 : offset + 8], "big")
+ offset += 8
+ if interfaces_count > (len(data) - offset) // 2:
+ return False
+ offset += interfaces_count * 2
+
+ offset = skip_java_members(data, offset)
+ if offset is None:
+ return False
+ offset = skip_java_members(data, offset)
+ if offset is None:
+ return False
+ result = read_java_u2(data, offset)
+ if result is None:
+ return False
+ attributes_count, offset = result
+ offset = skip_java_attributes(data, offset, attributes_count)
+ return offset == len(data)
+
+
+def native_binary_magic(source, size: int, name: str) -> str | None:
+ """Return the native executable format, excluding valid Java class
files."""
+ header = source.read(min(size, 64))
+ if header.startswith(b"\x7fELF"):
+ return "ELF"
+ if (
+ header.startswith(JAVA_CLASS_MAGIC)
+ and name.lower().endswith(".class")
+ ):
+ if size <= MAX_JAVA_CLASS_SIZE:
+ source.seek(0)
+ if is_java_class(source.read(MAX_JAVA_CLASS_SIZE + 1)):
+ return None
+ if header[:4] in MACHO_MAGICS:
+ return "Mach-O"
+ if header.startswith(b"MZ") and len(header) >= 64:
+ pe_offset = int.from_bytes(header[0x3C:0x40], "little")
+ if pe_offset <= size - 4:
+ source.seek(pe_offset)
+ if source.read(4) == b"PE\0\0":
+ return "PE"
+ return None
+
+
+def native_archive_entries(
+ archive: ZipFile, entries: dict[str, ZipInfo]
+) -> set[str]:
+ native_entries = set()
+ for name, info in entries.items():
+ if info.is_dir():
+ continue
+ with archive.open(info) as source:
+ magic = native_binary_magic(source, info.file_size, name)
Review Comment:
Fixed. Every JAR member is now read through EOF so its CRC is validated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
