JingsongLi commented on code in PR #7456: URL: https://github.com/apache/paimon/pull/7456#discussion_r3459138270
########## paimon-python/pypaimon/cli/cli_table_stream.py: ########## @@ -0,0 +1,216 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +""" +Table stream command for Paimon CLI. + +Continuously polls a table for new snapshots and prints rows as they arrive. +""" + +import json +import sys +from typing import Union + +from pypaimon.snapshot.snapshot_manager import SnapshotManager + + +def _parse_timestamp(value: str) -> int: + """Parse a human-readable datetime string to epoch milliseconds. + + Accepts ISO 8601, space-separated datetimes, date-only strings, and named + timezones (e.g. "2025-01-15T10:30:00 EST", "2025-01-15T10:30:00 Europe/London"). + Naive datetimes (no timezone) are treated as local machine time. + Returns epoch ms as an integer. + Raises ValueError on unrecognised format. + """ + from dateutil import parser as dateutil_parser + from dateutil.parser import ParserError + + try: + dt = dateutil_parser.parse(value) + except (ParserError, OverflowError): + raise ValueError( + f"Unrecognised timestamp format: '{value}'. " + "Expected formats: YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, " + "YYYY-MM-DDTHH:MM:SS, YYYY-MM-DDTHH:MM:SSZ, YYYY-MM-DDTHH:MM:SS+HH:MM, " + "or a datetime with a named timezone (e.g. '2025-01-15T10:30:00 EST')" + ) + + if dt.tzinfo is None: + # Treat naive datetime as local time + dt = dt.astimezone() + + return int(dt.timestamp() * 1000) + + +def parse_from_position(value: str, snapshot_manager: SnapshotManager) -> Union[str, int]: + """Resolve a --from value to a keyword or snapshot ID integer. + + Args: + value: The raw --from argument (keyword, integer string, or timestamp). + snapshot_manager: Used to resolve timestamps to snapshot IDs. + + Returns: + "latest", "earliest", or an integer snapshot ID. + + Raises: + ValueError: If the value is unrecognised or the timestamp precedes all snapshots. + """ + if value in ("latest", "earliest"): + return value + + # Pure integer → snapshot ID + if value.strip().isdigit(): + return int(value.strip()) + + # Anything containing '-', 'T', ':', or '/' is treated as a timestamp + if any(c in value for c in ("-", "T", ":", "/")): + epoch_ms = _parse_timestamp(value) + snapshot = snapshot_manager.earlier_or_equal_time_mills(epoch_ms) + if snapshot is None: + raise ValueError( + f"No snapshot found at or before '{value}'. " + "The timestamp may predate all stored snapshots." + ) + return snapshot.id Review Comment: `AsyncStreamingTableScan` treats a numeric `scan_from` as the next snapshot to emit, so resolving a timestamp with `earlier_or_equal_time_mills` and returning that same snapshot id makes `--from <timestamp>` include data from before the requested time. For example, with S1 at 10:00 and S2 at 11:00, `--from 10:30` resolves to S1 and then streams S1. Please either return `snapshot.id + 1` for timestamp inputs, or resolve to the first snapshot at/after the timestamp so timestamp starts do not replay an older commit. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
