JingsongLi commented on code in PR #7456: URL: https://github.com/apache/paimon/pull/7456#discussion_r3442690430
########## paimon-python/pypaimon/cli/cli_table_stream.py: ########## @@ -0,0 +1,216 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +""" +Table stream command for Paimon CLI. + +Continuously polls a table for new snapshots and prints rows as they arrive. +""" + +import json +import sys +from typing import Union + +from pypaimon.snapshot.snapshot_manager import SnapshotManager + + +def _parse_timestamp(value: str) -> int: + """Parse a human-readable datetime string to epoch milliseconds. + + Accepts ISO 8601, space-separated datetimes, date-only strings, and named + timezones (e.g. "2025-01-15T10:30:00 EST", "2025-01-15T10:30:00 Europe/London"). + Naive datetimes (no timezone) are treated as local machine time. + Returns epoch ms as an integer. + Raises ValueError on unrecognised format. + """ + from dateutil import parser as dateutil_parser + from dateutil.parser import ParserError + + try: + dt = dateutil_parser.parse(value) + except (ParserError, OverflowError): + raise ValueError( + f"Unrecognised timestamp format: '{value}'. " + "Expected formats: YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, " + "YYYY-MM-DDTHH:MM:SS, YYYY-MM-DDTHH:MM:SSZ, YYYY-MM-DDTHH:MM:SS+HH:MM, " + "or a datetime with a named timezone (e.g. '2025-01-15T10:30:00 EST')" + ) + + if dt.tzinfo is None: + # Treat naive datetime as local time + dt = dt.astimezone() + + return int(dt.timestamp() * 1000) + + +def parse_from_position(value: str, snapshot_manager: SnapshotManager) -> Union[str, int]: + """Resolve a --from value to a keyword or snapshot ID integer. + + Args: + value: The raw --from argument (keyword, integer string, or timestamp). + snapshot_manager: Used to resolve timestamps to snapshot IDs. + + Returns: + "latest", "earliest", or an integer snapshot ID. + + Raises: + ValueError: If the value is unrecognised or the timestamp precedes all snapshots. + """ + if value in ("latest", "earliest"): + return value + + # Pure integer → snapshot ID + if value.strip().isdigit(): + return int(value.strip()) + + # Anything containing '-', 'T', ':', or '/' is treated as a timestamp + if any(c in value for c in ("-", "T", ":", "/")): + epoch_ms = _parse_timestamp(value) + snapshot = snapshot_manager.earlier_or_equal_time_mills(epoch_ms) + if snapshot is None: + raise ValueError( + f"No snapshot found at or before '{value}'. " + "The timestamp may predate all stored snapshots." + ) + return snapshot.id + + raise ValueError( + f"Unrecognised --from value: '{value}'. " + "Use 'latest', 'earliest', a snapshot ID, or a timestamp." + ) + + +def cmd_table_stream(args): + """Execute the 'table stream' command. + + Continuously reads new rows from a Paimon table and prints them to stdout + until interrupted with Ctrl+C. + + Args: + args: Parsed command line arguments. + """ + from pypaimon.cli.cli import load_catalog_config, create_catalog + from pypaimon.table.file_store_table import FileStoreTable + + config = load_catalog_config(args.config) + catalog = create_catalog(config) + + table_identifier = args.table + parts = table_identifier.split('.') + if len(parts) != 2: + print( + f"Error: Invalid table identifier '{table_identifier}'. " + "Expected format: 'database.table'", + file=sys.stderr, + ) + sys.exit(1) + + try: + table = catalog.get_table(table_identifier) + except Exception as e: + print(f"Error: Failed to get table '{table_identifier}': {e}", file=sys.stderr) + sys.exit(1) + + available_fields = set(field.name for field in table.table_schema.fields) + + # --- Validate and build projection --- + user_columns = None + if args.select: + user_columns = [col.strip() for col in args.select.split(',')] + invalid_columns = [col for col in user_columns if col not in available_fields] + if invalid_columns: + print( + f"Error: Column(s) {invalid_columns} do not exist in table '{table_identifier}'.", + file=sys.stderr, + ) + sys.exit(1) + + # --- Parse WHERE predicate --- + predicate = None + if args.where: + from pypaimon.cli.where_parser import parse_where_clause + try: + predicate = parse_where_clause(args.where, table.table_schema.fields) + except ValueError as e: + print(f"Error: Invalid WHERE clause: {e}", file=sys.stderr) + sys.exit(1) + + # --- Resolve --from position --- + from_position = args.from_position + if from_position not in ("latest", "earliest") and not from_position.strip().isdigit(): + # Likely a timestamp — need snapshot_manager to resolve it + if not isinstance(table, FileStoreTable): + print( + "Error: Timestamp-based --from requires a FileStoreTable.", + file=sys.stderr, + ) + sys.exit(1) + snapshot_manager = table.snapshot_manager() + try: + from_position = parse_from_position(from_position, snapshot_manager) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + elif from_position.strip().isdigit(): + from_position = int(from_position.strip()) + + # --- Build StreamReadBuilder --- + builder = table.new_stream_read_builder() + + if user_columns: + builder = builder.with_projection(user_columns) Review Comment: When --select is combined with a --where predicate that references a non-selected column, this projection drops the predicate column from the read schema. The reader then cannot evaluate the row predicate and returns unfiltered rows. For example, --from earliest --select id --where "age > 18" returns both ids for rows (1, age=10) and (2, age=30). Please mirror table read here: add WHERE-referenced fields to the internal projection, then drop those extra columns from the printed output. ########## paimon-python/pypaimon/read/streaming_table_scan.py: ########## @@ -130,13 +134,28 @@ async def stream(self) -> AsyncGenerator[Plan, None]: Yields: Plan objects containing splits for reading """ - # Restore from consumer if available + # Restore from consumer if available (highest priority — overrides scan_from) if self.next_snapshot_id is None and self._consumer_manager: consumer = self._consumer_manager.consumer(self._consumer_id) if consumer: self.next_snapshot_id = consumer.next_snapshot - # Initial scan + # Resolve scan_from if no position has been set yet (no consumer restore) + if self.next_snapshot_id is None and self._scan_from is not None: + scan_from = self._scan_from + if scan_from == "earliest": + earliest_snapshot = self._snapshot_manager.try_get_earliest_snapshot() Review Comment: try_get_earliest_snapshot() raises RuntimeError when the table has no snapshots; it does not return None. That means table stream --from earliest on a newly created empty table exits before reaching the polling loop, despite the comment below. Please handle the no-snapshot case before or around this call so the stream waits for the first snapshot. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
