leaves12138 commented on code in PR #9825:
URL: https://github.com/apache/paimon/pull/9825#discussion_r4011411232
##########
paimon-python/pypaimon/utils/file_store_path_factory.py:
##########
@@ -149,6 +160,44 @@ def global_index_path_factory(self) -> 'IndexPathFactory':
self.global_index_external_path is not None,
)
+ def new_bucket_index_path(self, partition: Tuple, bucket: int, file_name:
str) -> Tuple[str, bool]:
+ """Return a new bucket index's path and whether to persist its
external location."""
+ if self.index_file_in_data_file_dir:
+ external = self.create_external_path_provider(partition, bucket)
+ if external is not None:
+ return external.get_next_external_data_path(file_name), True
+ # Python data directories historically use str(value) without
+ # escaping. Record the actual location when Java renders it
+ # differently, so its readers can find the DV beside those files.
+ return (f"{self.bucket_path(partition, bucket)}/{file_name}",
+ self._partition_path_requires_explicit_location(partition))
+ factory = self.global_index_path_factory()
+ return factory.to_path(file_name), factory.is_external_path()
+
+ def _partition_path_requires_explicit_location(self, partition: Tuple) ->
bool:
+ # FLOAT/DOUBLE formatting can differ from Python's float repr, and
+ # this factory does not carry field types to distinguish the two.
+ return (any(isinstance(value, float) for value in partition)
+ or self.relative_bucket_path(partition, 0) !=
self.relative_bucket_path(partition, 0, True))
+
+ def bucket_index_path(self, partition: Tuple, bucket: int, index_file,
file_io=None) -> str:
+ """Resolve an existing bucket index, including the legacy Python DV
layout."""
+ if index_file.external_path:
+ return index_file.external_path
+ legacy_path = f"{self.index_path()}/{index_file.file_name}"
+ if not self.index_file_in_data_file_dir:
+ return legacy_path
+ path = f"{self.bucket_path(partition, bucket,
True)}/{index_file.file_name}"
Review Comment:
[P2] Use schema-aware formatting for Java bucket-local DV paths
For a Java-written DV with `index-file-in-data-file-dir=true` and no
`external_path`, `canonical_partition=True` only escapes Python `str(value)`
and special-cases booleans; it does not reproduce Java's typed partition casts.
I verified Java's `InternalRowPartitionComputer`/`PartitionPathUtils` output
and reproduced read failures for both `FLOAT 0.1` (Java directory `p=0.1`,
Python looks under `p=0.10000000149011612`) and `TIMESTAMP(3) '2026-09-15
12:00:00'` (Java uses `p=2026-09-15 12%3A00%3A00.000`, Python omits `.000`).
None of the fallback paths locates the existing canonical DV, and reading
raises `FileNotFoundError`. Rust also intentionally rejects floating-point
partition formatting, so enabling native planning does not rescue the FLOAT
case.
Reproduction: extend `test_java_canonical_bucket_dv_without_external_path`
to these typed partitions, place the valid DV bytes at the Java-generated
canonical bucket path, clear its `external_path`, and remove the old
Python-path copy. Both Python reads fail instead of returning the surviving
rows. Please pass the partition field types into canonical path construction
and use Java-compatible formatting, retaining the current explicit-path and
legacy-path precedence, with regressions for Java-produced typed DV paths.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
