JingsongLi commented on code in PR #10351:
URL: https://github.com/apache/paimon/pull/10351#discussion_r4375292804
##########
paimon-python/pypaimon/common/uri_reader.py:
##########
@@ -179,7 +179,8 @@ def _new_reader(self, key: UriKey, parsed_uri: ParseResult)
-> UriReader:
uri_string = parsed_uri.geturl()
file_io = FileIO.get(uri_string, self.catalog_options)
self._owned_file_ios.append(file_io)
- return UriReader.from_file(file_io)
+ # This FileIO serves no table, so the URIs of row data it reads
come from origin.
+ return UriReader.from_file(file_io.origin_only())
Review Comment:
[P2] Resolve origin options before constructing a standalone URI FileIO
`FileIO.get(uri, catalog_options)` creates an ordinary `OssFileIO`, whose
inherited `origin_only()` returns itself; it does not install
`IoCacheRoutingFileIO`. With routing configured in the catalog,
`fs.oss.endpoint` pointing at the legacy cache, and `io-cache.origin.endpoint`
at the origin, this factory therefore still reads external row-data/BLOB URIs
from the cache. I verified this with real PyArrow HTTP requests: the same
external Parquet URI returned cached id `[99]`, while applying
`single_endpoint_options` before constructing the FileIO sent HEAD/GET to
origin and returned current ids `[1,2,3]`. Please resolve the effective origin
options before `FileIO.get`, or provide an equivalent origin-aware backend. The
existing test replaces `FileIO.get` with a routed wrapper, which hides this
production path.
##########
paimon-python/pypaimon/filesystem/io_cache_routing.py:
##########
@@ -0,0 +1,284 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Chooses the endpoint of each FileIO request from the io-cache.* keys of a
table token."""
+
+import copy
+import os
+import re
+from enum import Enum
+from typing import Callable, Dict, FrozenSet, Iterable, List, Mapping,
NamedTuple, Optional, Tuple
+
+from pypaimon.common.options.config import CatalogOptions, OssOptions
+from pypaimon.utils.file_type import FileType
+
+FS_OSS_ENDPOINT = OssOptions.OSS_ENDPOINT.key()
+FS_OSS_REGION = OssOptions.OSS_REGION.key()
+FS_OSS_PATH_STYLE = OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key()
+FS_OSS_HTTPS_ENABLE = "fs.oss.https.enable"
+DLF_OSS_ENDPOINT = CatalogOptions.DLF_OSS_ENDPOINT.key()
+IO_CACHE_ENABLED = CatalogOptions.IO_CACHE_ENABLED.key()
+IO_CACHE_TARGET_PREFIX = "io-cache.target."
+
+_TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*")
+_UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+# What follows a data file prefix: {uuid}-{count}.{extension}, also after the
postpone writer's
+# -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack
{uuid}.managed.blob.
+_DATA_FILE_SUFFIX = re.compile(r"(?:-u-.+-s-[0-9]+-w-)?" + _UUID +
r"-[0-9]+\..+|" + _UUID + r"\.managed\.blob")
+# A name that may be a data file under a prefix this routing has not seen yet.
+_DATA_FILE_NAME = re.compile(r".+?(?:" + _DATA_FILE_SUFFIX.pattern + r")")
+# Other types are routed only under the names Paimon writes (paimon-rust adds
-changelog to changelog
+# manifests); Format Table files may be replaced in place.
+_PAIMON_FILE_NAMES = {
+ FileType.META: re.compile(
+ r"(manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-"
+ _UUID
+ + r"(-changelog)?-[0-9]+(\.avro\.sidecar)?"),
+ FileType.BUCKET_INDEX: re.compile(r"index-" + _UUID + r"-[0-9]+"),
+ FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID +
r"\.index"),
+}
+_MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar"
+# JindoSDK cache client keys; the origin option set must not talk to a cache.
+_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.")
+
+
+class Op(Enum):
+ """FileIO operations; only READ, META, EXISTS, WRITE and TWO_PHASE_WRITE
can go to a target."""
+ READ = "read"
+ META = "meta"
+ EXISTS = "exists"
+ WRITE = "write"
+ LIST = "list"
+ DELETE = "delete"
+ RENAME = "rename"
+ MKDIRS = "mkdirs"
+ COPY = "copy"
+ ATOMIC_WRITE = "atomic-write"
+ TWO_PHASE_WRITE = "two-phase-write"
+ PRESIGN = "presign"
+
+
+def _flag(value) -> bool:
+ return value is not None and str(value).strip().lower() == "true"
+
+
+def _parse_policy(value) -> FrozenSet[Op]:
+ # read, meta, exists and write can use a target; none turns it off
+ tokens = {token.strip().lower() for token in str(value or "").split(",")}
+ if "none" in tokens:
+ return frozenset()
+ return frozenset(op for op in (Op.READ, Op.META, Op.EXISTS, Op.WRITE) if
op.value in tokens)
+
+
+def _parse_routes(value: str) -> List[Tuple[set, str]]:
+ # "types=target;...": the first rule containing a type picks its target
+ rules = []
+ for rule in value.lower().split(";"):
+ types, eq, target = rule.partition("=")
+ types = FileType.parse_whitelist(types) if eq and target.strip() else
set()
+ if types:
+ rules.append((types, target.strip()))
+ return rules
+
+
+def _declared_targets(options: Mapping[str, str]) -> Dict[str, Optional[str]]:
+ # io-cache.targets with io-cache.target.<name>.endpoint, else
io-cache.endpoint as "default"
+ names = options.get("io-cache.targets")
+ if names is None:
+ endpoint = options.get("io-cache.endpoint")
+ return {} if endpoint is None else {"default": endpoint}
+ endpoints = {}
+ for name in str(names).lower().split(","):
+ name = name.strip()
+ if _TARGET_NAME.fullmatch(name) and name not in endpoints:
+ endpoints[name] = options.get(IO_CACHE_TARGET_PREFIX + name +
".endpoint")
+ return endpoints
+
+
+def schema_file_prefixes(options) -> List[str]:
+ """The data-file.prefix and changelog-file.prefix a table schema sets."""
+ values = [options.get(key) for key in ("data-file.prefix",
"changelog-file.prefix")]
+ return [value for value in values if value and value.strip()]
+
+
+def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]:
+ """Data file name prefixes: data- and changelog-, plus the table's own
prefixes."""
+ prefixes = ["data-", "changelog-"]
+ for key in ("data-file.prefix", "changelog-file.prefix"):
+ value = options.get(key)
+ if value and value.strip() and value not in prefixes:
+ prefixes.append(value)
+ return tuple(prefixes)
+
+
+def _new_prefixes(prefixes: Iterable[str], known: Tuple[str, ...]) ->
Tuple[str, ...]:
+ return tuple(p for p in dict.fromkeys(prefixes) if p and p.strip() and p
not in known)
+
+
+def _table_root(path: Optional[str]) -> Optional[str]:
+ return path.rstrip("/") + "/" if path else None
+
+
+def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]:
+ """The type a cache may serve, or None when the file must be read from the
origin."""
+ path = path.rstrip("/")
+ name = os.path.basename(path)
+ if _is_temporary(name) or FileType.is_mutable(path):
+ return None
+ if _is_data_file_name(name, prefixes):
+ return FileType.FILE_INDEX if name.endswith(".index") else
FileType.DATA
+ # Sequential metadata may be read before it exists; cached NotFound would
hide new commits.
+ parent = os.path.basename(os.path.dirname(path))
+ if name.startswith(("snapshot-", "schema-")) or (
+ name.startswith("changelog-") and parent == "changelog"):
+ return None
+ file_type = FileType.classify(path)
+ # data files and their file indexes are recognized by _is_data_file_name
+ pattern = _PAIMON_FILE_NAMES.get(file_type)
+ return file_type if pattern is not None and pattern.fullmatch(name) else
None
+
+
+def _is_data_file_name(name: str, prefixes: Tuple[str, ...]) -> bool:
+ """Manifests, indexes and statistics share the uuid-count shape but have
no extension."""
+ if name.endswith(_MANIFEST_SIDECAR_SUFFIX):
+ return False
+ return any(name.startswith(p) and
_DATA_FILE_SUFFIX.fullmatch(name[len(p):]) for p in prefixes)
+
+
+def _is_temporary(name: str) -> bool:
+ return name.endswith(".tmp") or ".tmp-" in name or ".tmp." in name
+
+
+def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) ->
Dict[str, str]:
+ if not endpoint:
+ return options
+ # JindoSDK drops the scheme of a URL endpoint, so carry it in
fs.oss.https.enable.
+ scheme, separator, host = endpoint.partition("://")
+ if separator:
+ scheme = scheme.lower()
+ options[FS_OSS_HTTPS_ENABLE] = "true" if scheme == "https" else "false"
+ endpoint = scheme + separator + host
+ options[FS_OSS_ENDPOINT] = endpoint.rstrip("/")
+ return options
+
+
+def single_endpoint_options(options: Mapping[str, str],
+ catalog_options: Optional[Mapping[str, str]] =
None) -> Dict[str, str]:
+ """Options for a client that sends every request to one endpoint: the
origin's under io-cache routing,
+ unless catalog_options turn routing off (dlf.oss-endpoint)."""
+ routing = IoCacheRouting.create({**(catalog_options or {}), **options})
Review Comment:
[P2] Reapply the current catalog routing switch after token merging
The token can contain another instance’s catalog override:
`RESTTokenFileIO._TOKEN_CACHE` is shared by user/table, and `refresh_token`
puts its already-overridden `io-cache.enabled` into that cache. I used two real
RESTTokenFileIO instances for the same user/table: A with catalog false
refreshed once, then B with catalog true reused the same token. B’s Python
`_merged_options` correctly selected origin, but this token-last merge left
false in the native bridge’s data token, so native and Daft selected the legacy
cache. The Rust bridge was inspected and does not subsequently resolve io-cache
options or restore the catalog switch; this probe captured bridge arguments
rather than running Rust HTTP. Please enforce the current catalog’s explicit
enabled override after the general merge (also in Daft’s premerge), and cover
shared-token reuse between differing catalog settings.
##########
paimon-python/pypaimon/filesystem/pvfs.py:
##########
@@ -887,7 +888,12 @@ def _merge_token_with_catalog_options(self, token: dict)
-> dict:
dlf_oss_endpoint = self.options.get(CatalogOptions.DLF_OSS_ENDPOINT)
if dlf_oss_endpoint and dlf_oss_endpoint.strip():
merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint
- return merged_token
+ return merged_token
+ if self.options.contains(CatalogOptions.IO_CACHE_ENABLED):
+ merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str(
+ self.options.get(CatalogOptions.IO_CACHE_ENABLED)).lower()
+ # The OSS filesystem sends every request to one endpoint, so it cannot
use io-cache targets.
+ return single_endpoint_options(merged_token)
Review Comment:
[P2] Include catalog routing options when selecting the PVFS origin
This copies catalog `io-cache.enabled`, but drops catalog-only
`io-cache.policy`, target endpoint and `io-cache.origin.endpoint` before
calling `single_endpoint_options`. If the token contains credentials and the
legacy `fs.oss.endpoint`, routing cannot be recognized and PVFS keeps that
cache as its sole endpoint. Through the production `PVFS.cat_file` path,
catalog-only routing selected the old cache and returned `old-object`; putting
the identical routing options in the token selected origin and returned
`new-object`. The external OSS client boundary was mapped to real local files;
no routing/merge function was mocked. `_get_filesystem` passes this result
directly to the OSS backend, with no later catalog merge. Please merge catalog
and token routing options first, then apply the documented catalog enabled/DLF
overrides before selecting the origin.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]