JingsongLi commented on code in PR #10351:
URL: https://github.com/apache/paimon/pull/10351#discussion_r4375292804


##########
paimon-python/pypaimon/common/uri_reader.py:
##########
@@ -179,7 +179,8 @@ def _new_reader(self, key: UriKey, parsed_uri: ParseResult) 
-> UriReader:
             uri_string = parsed_uri.geturl()
             file_io = FileIO.get(uri_string, self.catalog_options)
             self._owned_file_ios.append(file_io)
-            return UriReader.from_file(file_io)
+            # This FileIO serves no table, so the URIs of row data it reads 
come from origin.
+            return UriReader.from_file(file_io.origin_only())

Review Comment:
   [P2] Resolve origin options before constructing a standalone URI FileIO
   
   `FileIO.get(uri, catalog_options)` creates an ordinary `OssFileIO`, whose 
inherited `origin_only()` returns itself; it does not install 
`IoCacheRoutingFileIO`. With routing configured in the catalog, 
`fs.oss.endpoint` pointing at the legacy cache, and `io-cache.origin.endpoint` 
at the origin, this factory therefore still reads external row-data/BLOB URIs 
from the cache. I verified this with real PyArrow HTTP requests: the same 
external Parquet URI returned cached id `[99]`, while applying 
`single_endpoint_options` before constructing the FileIO sent HEAD/GET to 
origin and returned current ids `[1,2,3]`. Please resolve the effective origin 
options before `FileIO.get`, or provide an equivalent origin-aware backend. The 
existing test replaces `FileIO.get` with a routed wrapper, which hides this 
production path.



##########
paimon-python/pypaimon/filesystem/io_cache_routing.py:
##########
@@ -0,0 +1,284 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Chooses the endpoint of each FileIO request from the io-cache.* keys of a 
table token."""
+
+import copy
+import os
+import re
+from enum import Enum
+from typing import Callable, Dict, FrozenSet, Iterable, List, Mapping, 
NamedTuple, Optional, Tuple
+
+from pypaimon.common.options.config import CatalogOptions, OssOptions
+from pypaimon.utils.file_type import FileType
+
+FS_OSS_ENDPOINT = OssOptions.OSS_ENDPOINT.key()
+FS_OSS_REGION = OssOptions.OSS_REGION.key()
+FS_OSS_PATH_STYLE = OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key()
+FS_OSS_HTTPS_ENABLE = "fs.oss.https.enable"
+DLF_OSS_ENDPOINT = CatalogOptions.DLF_OSS_ENDPOINT.key()
+IO_CACHE_ENABLED = CatalogOptions.IO_CACHE_ENABLED.key()
+IO_CACHE_TARGET_PREFIX = "io-cache.target."
+
+_TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*")
+_UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+# What follows a data file prefix: {uuid}-{count}.{extension}, also after the 
postpone writer's
+# -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack 
{uuid}.managed.blob.
+_DATA_FILE_SUFFIX = re.compile(r"(?:-u-.+-s-[0-9]+-w-)?" + _UUID + 
r"-[0-9]+\..+|" + _UUID + r"\.managed\.blob")
+# A name that may be a data file under a prefix this routing has not seen yet.
+_DATA_FILE_NAME = re.compile(r".+?(?:" + _DATA_FILE_SUFFIX.pattern + r")")
+# Other types are routed only under the names Paimon writes (paimon-rust adds 
-changelog to changelog
+# manifests); Format Table files may be replaced in place.
+_PAIMON_FILE_NAMES = {
+    FileType.META: re.compile(
+        r"(manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-" 
+ _UUID
+        + r"(-changelog)?-[0-9]+(\.avro\.sidecar)?"),
+    FileType.BUCKET_INDEX: re.compile(r"index-" + _UUID + r"-[0-9]+"),
+    FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID + 
r"\.index"),
+}
+_MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar"
+# JindoSDK cache client keys; the origin option set must not talk to a cache.
+_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.")
+
+
+class Op(Enum):
+    """FileIO operations; only READ, META, EXISTS, WRITE and TWO_PHASE_WRITE 
can go to a target."""
+    READ = "read"
+    META = "meta"
+    EXISTS = "exists"
+    WRITE = "write"
+    LIST = "list"
+    DELETE = "delete"
+    RENAME = "rename"
+    MKDIRS = "mkdirs"
+    COPY = "copy"
+    ATOMIC_WRITE = "atomic-write"
+    TWO_PHASE_WRITE = "two-phase-write"
+    PRESIGN = "presign"
+
+
+def _flag(value) -> bool:
+    return value is not None and str(value).strip().lower() == "true"
+
+
+def _parse_policy(value) -> FrozenSet[Op]:
+    # read, meta, exists and write can use a target; none turns it off
+    tokens = {token.strip().lower() for token in str(value or "").split(",")}
+    if "none" in tokens:
+        return frozenset()
+    return frozenset(op for op in (Op.READ, Op.META, Op.EXISTS, Op.WRITE) if 
op.value in tokens)
+
+
+def _parse_routes(value: str) -> List[Tuple[set, str]]:
+    # "types=target;...": the first rule containing a type picks its target
+    rules = []
+    for rule in value.lower().split(";"):
+        types, eq, target = rule.partition("=")
+        types = FileType.parse_whitelist(types) if eq and target.strip() else 
set()
+        if types:
+            rules.append((types, target.strip()))
+    return rules
+
+
+def _declared_targets(options: Mapping[str, str]) -> Dict[str, Optional[str]]:
+    # io-cache.targets with io-cache.target.<name>.endpoint, else 
io-cache.endpoint as "default"
+    names = options.get("io-cache.targets")
+    if names is None:
+        endpoint = options.get("io-cache.endpoint")
+        return {} if endpoint is None else {"default": endpoint}
+    endpoints = {}
+    for name in str(names).lower().split(","):
+        name = name.strip()
+        if _TARGET_NAME.fullmatch(name) and name not in endpoints:
+            endpoints[name] = options.get(IO_CACHE_TARGET_PREFIX + name + 
".endpoint")
+    return endpoints
+
+
+def schema_file_prefixes(options) -> List[str]:
+    """The data-file.prefix and changelog-file.prefix a table schema sets."""
+    values = [options.get(key) for key in ("data-file.prefix", 
"changelog-file.prefix")]
+    return [value for value in values if value and value.strip()]
+
+
+def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]:
+    """Data file name prefixes: data- and changelog-, plus the table's own 
prefixes."""
+    prefixes = ["data-", "changelog-"]
+    for key in ("data-file.prefix", "changelog-file.prefix"):
+        value = options.get(key)
+        if value and value.strip() and value not in prefixes:
+            prefixes.append(value)
+    return tuple(prefixes)
+
+
+def _new_prefixes(prefixes: Iterable[str], known: Tuple[str, ...]) -> 
Tuple[str, ...]:
+    return tuple(p for p in dict.fromkeys(prefixes) if p and p.strip() and p 
not in known)
+
+
+def _table_root(path: Optional[str]) -> Optional[str]:
+    return path.rstrip("/") + "/" if path else None
+
+
+def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]:
+    """The type a cache may serve, or None when the file must be read from the 
origin."""
+    path = path.rstrip("/")
+    name = os.path.basename(path)
+    if _is_temporary(name) or FileType.is_mutable(path):
+        return None
+    if _is_data_file_name(name, prefixes):
+        return FileType.FILE_INDEX if name.endswith(".index") else 
FileType.DATA
+    # Sequential metadata may be read before it exists; cached NotFound would 
hide new commits.
+    parent = os.path.basename(os.path.dirname(path))
+    if name.startswith(("snapshot-", "schema-")) or (
+            name.startswith("changelog-") and parent == "changelog"):
+        return None
+    file_type = FileType.classify(path)
+    # data files and their file indexes are recognized by _is_data_file_name
+    pattern = _PAIMON_FILE_NAMES.get(file_type)
+    return file_type if pattern is not None and pattern.fullmatch(name) else 
None
+
+
+def _is_data_file_name(name: str, prefixes: Tuple[str, ...]) -> bool:
+    """Manifests, indexes and statistics share the uuid-count shape but have 
no extension."""
+    if name.endswith(_MANIFEST_SIDECAR_SUFFIX):
+        return False
+    return any(name.startswith(p) and 
_DATA_FILE_SUFFIX.fullmatch(name[len(p):]) for p in prefixes)
+
+
+def _is_temporary(name: str) -> bool:
+    return name.endswith(".tmp") or ".tmp-" in name or ".tmp." in name
+
+
+def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> 
Dict[str, str]:
+    if not endpoint:
+        return options
+    # JindoSDK drops the scheme of a URL endpoint, so carry it in 
fs.oss.https.enable.
+    scheme, separator, host = endpoint.partition("://")
+    if separator:
+        scheme = scheme.lower()
+        options[FS_OSS_HTTPS_ENABLE] = "true" if scheme == "https" else "false"
+        endpoint = scheme + separator + host
+    options[FS_OSS_ENDPOINT] = endpoint.rstrip("/")
+    return options
+
+
+def single_endpoint_options(options: Mapping[str, str],
+                            catalog_options: Optional[Mapping[str, str]] = 
None) -> Dict[str, str]:
+    """Options for a client that sends every request to one endpoint: the 
origin's under io-cache routing,
+    unless catalog_options turn routing off (dlf.oss-endpoint)."""
+    routing = IoCacheRouting.create({**(catalog_options or {}), **options})

Review Comment:
   [P2] Reapply the current catalog routing switch after token merging
   
   The token can contain another instance’s catalog override: 
`RESTTokenFileIO._TOKEN_CACHE` is shared by user/table, and `refresh_token` 
puts its already-overridden `io-cache.enabled` into that cache. I used two real 
RESTTokenFileIO instances for the same user/table: A with catalog false 
refreshed once, then B with catalog true reused the same token. B’s Python 
`_merged_options` correctly selected origin, but this token-last merge left 
false in the native bridge’s data token, so native and Daft selected the legacy 
cache. The Rust bridge was inspected and does not subsequently resolve io-cache 
options or restore the catalog switch; this probe captured bridge arguments 
rather than running Rust HTTP. Please enforce the current catalog’s explicit 
enabled override after the general merge (also in Daft’s premerge), and cover 
shared-token reuse between differing catalog settings.



##########
paimon-python/pypaimon/filesystem/pvfs.py:
##########
@@ -887,7 +888,12 @@ def _merge_token_with_catalog_options(self, token: dict) 
-> dict:
         dlf_oss_endpoint = self.options.get(CatalogOptions.DLF_OSS_ENDPOINT)
         if dlf_oss_endpoint and dlf_oss_endpoint.strip():
             merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint
-        return merged_token
+            return merged_token
+        if self.options.contains(CatalogOptions.IO_CACHE_ENABLED):
+            merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str(
+                self.options.get(CatalogOptions.IO_CACHE_ENABLED)).lower()
+        # The OSS filesystem sends every request to one endpoint, so it cannot 
use io-cache targets.
+        return single_endpoint_options(merged_token)

Review Comment:
   [P2] Include catalog routing options when selecting the PVFS origin
   
   This copies catalog `io-cache.enabled`, but drops catalog-only 
`io-cache.policy`, target endpoint and `io-cache.origin.endpoint` before 
calling `single_endpoint_options`. If the token contains credentials and the 
legacy `fs.oss.endpoint`, routing cannot be recognized and PVFS keeps that 
cache as its sole endpoint. Through the production `PVFS.cat_file` path, 
catalog-only routing selected the old cache and returned `old-object`; putting 
the identical routing options in the token selected origin and returned 
`new-object`. The external OSS client boundary was mapped to real local files; 
no routing/merge function was mocked. `_get_filesystem` passes this result 
directly to the OSS backend, with no later catalog merge. Please merge catalog 
and token routing options first, then apply the documented catalog enabled/DLF 
overrides before selecting the origin.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to