github-actions[bot] commented on code in PR #41423:
URL: https://github.com/apache/beam/pull/41423#discussion_r4596618872
##########
sdks/python/apache_beam/io/requestresponse.py:
##########
@@ -818,24 +832,45 @@ def expand(
if isinstance(self._throttler, DefaultThrottler):
# DefaultThrottler applies throttling in the DoFn of
# Call PTransform.
- responses = (
- inputs
- | _Call(
- caller=self._caller,
- timeout=self._timeout,
- should_backoff=self._should_backoff,
- repeater=self._repeater,
- throttler=self._throttler))
+ call_transform = _Call(
+ caller=self._caller,
+ timeout=self._timeout,
+ should_backoff=self._should_backoff,
+ repeater=self._repeater,
+ throttler=self._throttler)
else:
# No throttling mechanism. The requests are made to the external source
# as they come.
- responses = (
- inputs
- | _Call(
- caller=self._caller,
- timeout=self._timeout,
- should_backoff=self._should_backoff,
- repeater=self._repeater))
+ call_transform = _Call(
+ caller=self._caller,
+ timeout=self._timeout,
+ should_backoff=self._should_backoff,
+ repeater=self._repeater)
+
+ if self._exception_handling_args is not None:
+ main_tag = self._exception_handling_args.get('main_tag', 'good')
+ dead_letter_tag = self._exception_handling_args.get(
+ 'dead_letter_tag', 'bad')
+ call_result = inputs | call_transform.with_exception_handling(
+ **self._exception_handling_args)
+ responses = call_result[main_tag]
+ errors = call_result[dead_letter_tag]
+
+ if self._batching_kwargs:
Review Comment:
**🟡 Medium**: When `self._cache` is enabled alongside
`with_exception_handling`, cache misses or errors inside
`self._cache.get_read()` and `self._cache.get_write()` are not protected by
`with_exception_handling`, and more importantly `cached_responses` may have a
mismatched coder/type hint or fail `beam.Flatten()` if `responses` is empty or
typed differently by `ParDo.with_exception_handling`. Additionally, if
`self._cache` is configured, `responses` written to cache in line 868 includes
empty miss responses (e.g., `(request, beam.Row())`), which is existing
behavior, but if an exception occurs in `_Call`, `cached_responses` still need
to be flattened even when `self._batching_kwargs` is used. Consider extracting
the post-call batch flattening and cache write/flatten logic so it isn't
duplicated between the exception-handling and non-exception-handling branches.
##########
sdks/python/apache_beam/transforms/enrichment.py:
##########
@@ -55,32 +64,129 @@ def has_valid_redis_address(host: str, port: int) -> bool:
return False
-def cross_join(left: dict[str, Any], right: dict[str, Any]) -> beam.Row:
+def cross_join(
+ left: dict[str, Any],
+ right: dict[str, Any],
+ prefix: Optional[str] = None) -> beam.Row:
"""performs a cross join on two `dict` objects.
Joins the columns of the right row onto the left row.
Args:
left (dict[str, Any]): input request dictionary.
right (dict[str, Any]): response dictionary from the API.
+ prefix (Optional[str]): optional prefix to prepend to keys from `right`
+ to avoid column name collisions.
Returns:
`beam.Row` containing the merged columns.
"""
for k, v in right.items():
- if k not in left:
+ target_key = f"{prefix}{k}" if prefix else k
+ if target_key not in left:
# Don't override the values in left.
- left[k] = v
- elif left[k] != v:
+ left[target_key] = v
+ elif left[target_key] != v:
_LOGGER.warning(
'%s exists in the input row as well the row fetched '
'from API but have different values - %s and %s. Using the input '
'value (%s) for the enriched row. You can override this behavior by '
'passing a custom `join_fn` to Enrichment transform.' %
- (k, left[k], v, left[k]))
+ (target_key, left[target_key], v, left[target_key]))
return beam.Row(**left)
+class _EnrichmentJoinFn(beam.DoFn):
+ """DoFn that joins `(request, response)` tuples with `on_miss` and
`prefix`."""
+ def __init__(
+ self,
+ join_fn: JoinFn,
+ use_custom_types: bool = False,
+ on_miss: str = 'null',
+ prefix: Optional[str] = None,
+ null_fields: Optional[list[str]] = None,
+ dead_letter_tag: Optional[str] = None,
+ ):
+ self._join_fn = join_fn
+ self._use_custom_types = use_custom_types
+ self._on_miss = on_miss
+ self._prefix = prefix
+ self._null_fields = list(null_fields) if null_fields else []
+ self._dead_letter_tag = dead_letter_tag
+
+ @staticmethod
+ def _is_miss(response: Any) -> bool:
+ if response is None:
+ return True
+ if hasattr(response, '_asdict'):
+ return len(response._asdict()) == 0
+ if isinstance(response, Mapping):
+ return len(response) == 0
+ return False
+
+ @staticmethod
+ def _to_dict(obj: Any) -> dict[str, Any]:
+ if hasattr(obj, '_asdict'):
+ return dict(obj._asdict())
+ return dict(obj)
+
+ def process(self, element: tuple[Any, Any], *args, **kwargs):
+ request, response = element
+ if self._is_miss(response):
+ if self._on_miss == 'drop':
+ return
+ if self._on_miss == 'fail':
+ raise KeyError(f"Enrichment lookup missed for element: {request}")
+ if self._on_miss == 'dlq':
+ try:
+ raise KeyError(f"Enrichment lookup missed for element: {request}")
+ except Exception as exn:
+ if self._dead_letter_tag:
+ yield pvalue.TaggedOutput(
+ self._dead_letter_tag,
+ (
+ request,
+ (
+ type(exn),
+ repr(exn),
+ traceback.format_exception(*sys.exc_info()))))
+ return
+ raise
+
+ # on_miss == 'null' (left join)
+ if self._use_custom_types:
+ yield self._join_fn(request, response)
+ return
+ left_dict = self._to_dict(request)
+ right_dict: dict[str, Any] = {f: None for f in self._null_fields}
Review Comment:
**🟠High**: When `self._on_miss == 'null'` and the handler does not provide
a static output schema (e.g. `BigQueryEnrichmentHandler`,
`BigTableEnrichmentHandler`, `VertexAIFeatureStoreEnrichmentHandler`, or
`FileEnrichmentHandler` without `select_fields` when the file isn't accessible
at graph construction time), `self._null_fields` is `[]`. In that case, a
lookup miss yields `beam.Row(**left_dict)` containing **only** the input
columns, whereas a lookup hit yields `beam.Row` containing the input columns
**plus** the enriched columns. Having rows with different sets of fields in the
same output `PCollection` breaks Beam schema consistency and downstream
transforms (such as Beam YAML SQL/Filter/Map or schema coders) that expect a
uniform `Row` schema across all elements.
##########
sdks/python/apache_beam/transforms/enrichment_handlers/file.py:
##########
@@ -0,0 +1,481 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import csv
+import io
+import json
+import logging
+import math
+import os
+from typing import Any
+from typing import Optional
+
+import apache_beam as beam
+from apache_beam.io.filesystems import FileSystems
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
+from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel
+from apache_beam.utils import shared
+
+try:
+ import pyarrow as pa
+ import pyarrow.parquet as pq
+except ImportError:
+ pa = None # type: ignore[assignment]
+ pq = None # type: ignore[assignment]
+
+__all__ = [
+ 'FileEnrichmentHandler',
+]
+
+_LOGGER = logging.getLogger(__name__)
+
+_FORMAT_ALIASES = {
+ 'parquet': 'parquet',
+ 'pq': 'parquet',
+ 'csv': 'csv',
+ 'tsv': 'tsv',
+ 'json': 'json',
+ 'jsonl': 'jsonl',
+ 'ndjson': 'jsonl',
+}
+
+
+class _LookupTable:
+ """Weak-referenceable container for in-memory reference records and
schema."""
+ def __init__(
+ self,
+ records: dict[Any, dict[str, Any]],
+ str_records: dict[str, dict[str, Any]],
+ schema: list[tuple[str, Any]],
+ ):
+ self.records = records
+ self.str_records = str_records
+ self.schema = schema
+
+ def get(self, key: Any) -> Optional[dict[str, Any]]:
+ if key is None:
+ return self.records.get(None)
+ if key in self.records:
+ return self.records[key]
+ str_key = str(key)
+ if str_key in self.str_records:
+ return self.str_records[str_key]
+ return None
+
+
+def _arrow_type_to_python_type(arrow_type: Any) -> Any:
+ if pa is None:
+ return Any
+ if pa.types.is_boolean(arrow_type):
+ return bool
+ if pa.types.is_integer(arrow_type):
+ return int
+ if pa.types.is_floating(arrow_type) or pa.types.is_decimal(arrow_type):
+ return float
+ if (pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type) or
+ pa.types.is_unicode(arrow_type)):
+ return str
+ if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type):
+ return bytes
+ if pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type):
+ elem_type = _arrow_type_to_python_type(arrow_type.value_type)
+ return list[elem_type] # type: ignore[valid-type]
+ return Any
+
+
+def _is_int_string(val: str) -> bool:
+ s = val.strip()
+ if not s:
+ return False
+ unsigned = s[1:] if s[0] in ('+', '-') else s
+ if not unsigned.isdigit():
+ return False
+ if len(unsigned) > 1 and unsigned.startswith('0'):
+ return False
+ return True
+
+
+def _is_float_string(val: str) -> bool:
+ s = val.strip()
+ if not s:
+ return False
+ try:
+ parsed = float(s)
+ return not (math.isnan(parsed) or math.isinf(parsed))
+ except ValueError:
+ return False
+
+
+def _infer_and_convert_csv_column(
+ raw_values: list[Optional[str]],
+ parse_types: bool = True) -> tuple[list[Any], Any]:
+ has_null = any(v is None or v == '' for v in raw_values)
+ non_empty = [v for v in raw_values if v is not None and v != '']
+ if not non_empty or not parse_types:
+ converted = [None if (v is None or v == '') else v for v in raw_values]
+ base_type: Any = str
+ return converted, (Optional[base_type] if has_null else base_type)
+
+ if all(v.strip().lower() in ('true', 'false') for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else (v.strip().lower() == 'true')
+ for v in raw_values
+ ]
+ base_type = bool
+ elif all(_is_int_string(v) for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else int(v.strip()) for v in raw_values
+ ]
+ base_type = int
+ elif all(_is_float_string(v) for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else float(v.strip()) for v in
raw_values
+ ]
+ base_type = float
+ else:
+ converted = [None if (v is None or v == '') else v for v in raw_values]
+ base_type = str
+
+ return converted, (Optional[base_type] if has_null else base_type)
+
+
+def _infer_python_column_type(values: list[Any]) -> Any:
+ has_null = any(v is None for v in values)
+ non_null = [v for v in values if v is not None]
+ if not non_null:
+ return Optional[Any]
+
+ types_seen = {type(v) for v in non_null}
+ if types_seen == {bool}:
+ base_type: Any = bool
+ elif types_seen == {int}:
+ base_type = int
+ elif types_seen <= {int, float}:
+ base_type = float
+ elif types_seen == {str}:
+ base_type = str
+ elif types_seen == {bytes}:
+ base_type = bytes
+ else:
+ base_type = Any
+
+ return Optional[base_type] if has_null else base_type
+
+
+class FileEnrichmentHandler(EnrichmentSourceHandler[beam.Row, beam.Row]):
+ """Enrichment handler for file-backed reference tables (Parquet, CSV,
JSON/JSONL).
+
+ Loads a reference file via :class:`apache_beam.io.filesystems.FileSystems`
+ in worker ``__enter__`` and shares the in-memory lookup dictionary across
+ worker threads via :class:`apache_beam.utils.shared.Shared`.
+
+ Args:
+ path (str): Path or file pattern to the reference file (e.g. `.parquet`,
+ `.csv`, `.json`, or `.jsonl` on local disk, GCS, or S3).
+ join_key (str): Key column name in the reference file (and in the input
+ `beam.Row` unless `row_key` is specified separately).
+ row_key (Optional[str]): Field name in the input `beam.Row` to use for
+ lookup. Defaults to `join_key`.
+ select_fields (Optional[list[str]]): List of column names to select from
+ the reference file. If `None`, all columns other than `join_key` are
+ returned.
+ file_format (Optional[str]): Explicit file format (`"parquet"`, `"csv"`,
+ `"json"`, or `"jsonl"`). If omitted, inferred from `path`.
+ format (Optional[str]): Alias for `file_format`.
+ delimiter (str): Field delimiter for CSV files. Defaults to `","`.
+ encoding (str): Text encoding for CSV/JSON/JSONL files. Defaults to
+ `"utf-8"`.
+ parse_csv_types (bool): Whether to infer primitive types (`int`, `float`,
+ `bool`, `str`) for CSV columns. Defaults to `True`.
+ exception_level (ExceptionLevel): Controls behavior when a key is not
+ found in the reference file. Defaults to `ExceptionLevel.QUIET`.
+ """
+ def __init__(
+ self,
+ path: str,
+ join_key: str = "",
+ *,
+ row_key: Optional[str] = None,
+ select_fields: Optional[list[str]] = None,
+ file_format: Optional[str] = None,
+ format: Optional[str] = None, # pylint: disable=redefined-builtin
+ delimiter: str = ',',
+ encoding: str = 'utf-8',
+ parse_csv_types: bool = True,
+ exception_level: ExceptionLevel = ExceptionLevel.QUIET,
+ ):
+ if not path:
+ raise ValueError("`path` must be provided for FileEnrichmentHandler.")
+ if not join_key and not row_key:
+ raise ValueError(
+ "Please specify `join_key` (or `row_key`) to identify the lookup "
+ "key column.")
+ self._path = path
+ self._join_key = join_key or (row_key or "")
+ self._row_key = row_key or self._join_key
+ self._select_fields = list(
+ select_fields) if select_fields is not None else None
+ raw_format = file_format or format or self._infer_format(path)
+ normalized_format = _FORMAT_ALIASES.get(raw_format.lower())
+ if not normalized_format:
+ raise ValueError(
+ f"Unsupported file format '{raw_format}'. Supported formats are: "
+ "parquet, csv, json, jsonl.")
+ self._format = normalized_format
+ self._delimiter = '\t' if (
+ self._format == 'tsv' and delimiter == ',') else delimiter
+ self._encoding = encoding
+ self._parse_csv_types = parse_csv_types
+ self._exception_level = exception_level
+ self._shared_handle = shared.Shared()
+ self._table: Optional[_LookupTable] = None
+ self._cached_schema: Optional[list[tuple[str, Any]]] = None
+
+ @staticmethod
+ def _infer_format(path: str) -> str:
+ clean_path = path.rstrip('*')
+ _, ext = os.path.splitext(clean_path)
+ ext = ext.lstrip('.').lower()
+ if ext in ('gz', 'bz2', 'zst', 'snappy'):
+ _, ext = os.path.splitext(clean_path[:-len(ext) - 1])
+ ext = ext.lstrip('.').lower()
+ if not ext:
+ raise ValueError(
+ f"Could not infer file format from path '{path}'. Please specify "
+ "`file_format` explicitly.")
+ return ext
+
+ def _resolve_paths(self) -> list[str]:
+ try:
+ match_results = FileSystems.match([self._path])
+ if match_results and match_results[0].metadata_list:
+ return [m.path for m in match_results[0].metadata_list]
+ except Exception:
+ pass
+ if FileSystems.exists(self._path):
+ return [self._path]
+ raise FileNotFoundError(f"Reference file not found at path: {self._path}")
+
+ def _determine_target_columns(self, all_columns: list[str]) -> list[str]:
+ if self._join_key not in all_columns:
+ raise KeyError(
+ f"join_key '{self._join_key}' not found in reference file columns: "
+ f"{all_columns}")
+ if self._select_fields is not None:
+ missing = [c for c in self._select_fields if c not in all_columns]
+ if missing:
+ raise KeyError(
+ f"select_fields {missing} not found in reference file columns: "
+ f"{all_columns}")
+ return list(self._select_fields)
+ return [c for c in all_columns if c != self._join_key]
+
+ def _load_parquet(self, file_paths: list[str]) -> _LookupTable:
+ if pa is None or pq is None:
+ raise ImportError(
+ "pyarrow is required to read Parquet files in
FileEnrichmentHandler.")
+ tables = []
+ for file_path in file_paths:
+ with FileSystems.open(file_path, 'rb') as f:
+ data = f.read()
+ tables.append(pq.read_table(pa.BufferReader(data)))
+
+ combined = pa.concat_tables(tables) if len(tables) > 1 else tables[0]
+ target_cols = self._determine_target_columns(combined.column_names)
+
+ rows = combined.to_pylist()
+ records: dict[Any, dict[str, Any]] = {}
+ str_records: dict[str, dict[str, Any]] = {}
+ col_values: dict[str, list[Any]] = {c: [] for c in target_cols}
+
+ for row in rows:
+ key = row.get(self._join_key)
+ projected = {c: row.get(c) for c in target_cols}
+ for c in target_cols:
+ col_values[c].append(projected[c])
+ records[key] = projected
+ if key is not None:
+ str_records[str(key)] = projected
+
+ schema: list[tuple[str, Any]] = []
+ arrow_schema = combined.schema
+ for c in target_cols:
+ field = arrow_schema.field(c)
+ py_type = _arrow_type_to_python_type(field.type)
+ has_null = any(v is None for v in col_values[c])
+ if has_null and py_type is not Any:
+ py_type = Optional[py_type]
+ schema.append((c, py_type))
+
+ return _LookupTable(records, str_records, schema)
+
+ def _load_csv(self, file_paths: list[str]) -> _LookupTable:
+ raw_rows: list[dict[str, Optional[str]]] = []
+ fieldnames: list[str] = []
+ for file_path in file_paths:
+ with FileSystems.open(file_path, 'rb') as f:
Review Comment:
**🟡 Medium**: If multiple CSV files are matched by a glob pattern (e.g.,
`part-*.csv`) and have columns in different orders or a first file that is
empty (header-only or 0 bytes), `fieldnames` is only captured from the first
non-empty `reader.fieldnames`, and `_determine_target_columns` isn't validated
against subsequent files. Also, decoding with `'utf-8'` when CSV files contain
a UTF-8 BOM (`\ufeff`) will cause the first column name in `fieldnames` to
include `'\ufeff'`, causing `_determine_target_columns` to fail with a
confusing `KeyError` when `join_key` is the first column. Using `'utf-8-sig'`
as the default or stripping BOM when `self._encoding.lower() in ('utf-8',
'utf8')` prevents this common CSV pitfall.
##########
sdks/python/apache_beam/transforms/enrichment_handlers/file.py:
##########
@@ -0,0 +1,481 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import csv
+import io
+import json
+import logging
+import math
+import os
+from typing import Any
+from typing import Optional
+
+import apache_beam as beam
+from apache_beam.io.filesystems import FileSystems
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
+from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel
+from apache_beam.utils import shared
+
+try:
+ import pyarrow as pa
+ import pyarrow.parquet as pq
+except ImportError:
+ pa = None # type: ignore[assignment]
+ pq = None # type: ignore[assignment]
+
+__all__ = [
+ 'FileEnrichmentHandler',
+]
+
+_LOGGER = logging.getLogger(__name__)
+
+_FORMAT_ALIASES = {
+ 'parquet': 'parquet',
+ 'pq': 'parquet',
+ 'csv': 'csv',
+ 'tsv': 'tsv',
+ 'json': 'json',
+ 'jsonl': 'jsonl',
+ 'ndjson': 'jsonl',
+}
+
+
+class _LookupTable:
+ """Weak-referenceable container for in-memory reference records and
schema."""
+ def __init__(
+ self,
+ records: dict[Any, dict[str, Any]],
+ str_records: dict[str, dict[str, Any]],
+ schema: list[tuple[str, Any]],
+ ):
+ self.records = records
+ self.str_records = str_records
+ self.schema = schema
+
+ def get(self, key: Any) -> Optional[dict[str, Any]]:
+ if key is None:
+ return self.records.get(None)
+ if key in self.records:
+ return self.records[key]
+ str_key = str(key)
+ if str_key in self.str_records:
+ return self.str_records[str_key]
+ return None
+
+
+def _arrow_type_to_python_type(arrow_type: Any) -> Any:
+ if pa is None:
+ return Any
+ if pa.types.is_boolean(arrow_type):
+ return bool
+ if pa.types.is_integer(arrow_type):
+ return int
+ if pa.types.is_floating(arrow_type) or pa.types.is_decimal(arrow_type):
+ return float
+ if (pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type) or
+ pa.types.is_unicode(arrow_type)):
+ return str
+ if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type):
+ return bytes
+ if pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type):
+ elem_type = _arrow_type_to_python_type(arrow_type.value_type)
+ return list[elem_type] # type: ignore[valid-type]
+ return Any
+
+
+def _is_int_string(val: str) -> bool:
+ s = val.strip()
+ if not s:
+ return False
+ unsigned = s[1:] if s[0] in ('+', '-') else s
+ if not unsigned.isdigit():
+ return False
+ if len(unsigned) > 1 and unsigned.startswith('0'):
+ return False
+ return True
+
+
+def _is_float_string(val: str) -> bool:
+ s = val.strip()
+ if not s:
+ return False
+ try:
+ parsed = float(s)
+ return not (math.isnan(parsed) or math.isinf(parsed))
+ except ValueError:
+ return False
+
+
+def _infer_and_convert_csv_column(
+ raw_values: list[Optional[str]],
+ parse_types: bool = True) -> tuple[list[Any], Any]:
+ has_null = any(v is None or v == '' for v in raw_values)
+ non_empty = [v for v in raw_values if v is not None and v != '']
+ if not non_empty or not parse_types:
+ converted = [None if (v is None or v == '') else v for v in raw_values]
+ base_type: Any = str
+ return converted, (Optional[base_type] if has_null else base_type)
+
+ if all(v.strip().lower() in ('true', 'false') for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else (v.strip().lower() == 'true')
+ for v in raw_values
+ ]
+ base_type = bool
+ elif all(_is_int_string(v) for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else int(v.strip()) for v in raw_values
+ ]
+ base_type = int
+ elif all(_is_float_string(v) for v in non_empty):
+ converted = [
+ None if (v is None or v == '') else float(v.strip()) for v in
raw_values
+ ]
+ base_type = float
+ else:
+ converted = [None if (v is None or v == '') else v for v in raw_values]
+ base_type = str
+
+ return converted, (Optional[base_type] if has_null else base_type)
+
+
+def _infer_python_column_type(values: list[Any]) -> Any:
+ has_null = any(v is None for v in values)
+ non_null = [v for v in values if v is not None]
+ if not non_null:
+ return Optional[Any]
+
+ types_seen = {type(v) for v in non_null}
+ if types_seen == {bool}:
+ base_type: Any = bool
+ elif types_seen == {int}:
+ base_type = int
+ elif types_seen <= {int, float}:
+ base_type = float
+ elif types_seen == {str}:
+ base_type = str
+ elif types_seen == {bytes}:
+ base_type = bytes
+ else:
+ base_type = Any
+
+ return Optional[base_type] if has_null else base_type
+
+
+class FileEnrichmentHandler(EnrichmentSourceHandler[beam.Row, beam.Row]):
+ """Enrichment handler for file-backed reference tables (Parquet, CSV,
JSON/JSONL).
+
+ Loads a reference file via :class:`apache_beam.io.filesystems.FileSystems`
+ in worker ``__enter__`` and shares the in-memory lookup dictionary across
+ worker threads via :class:`apache_beam.utils.shared.Shared`.
+
+ Args:
+ path (str): Path or file pattern to the reference file (e.g. `.parquet`,
+ `.csv`, `.json`, or `.jsonl` on local disk, GCS, or S3).
+ join_key (str): Key column name in the reference file (and in the input
+ `beam.Row` unless `row_key` is specified separately).
+ row_key (Optional[str]): Field name in the input `beam.Row` to use for
+ lookup. Defaults to `join_key`.
+ select_fields (Optional[list[str]]): List of column names to select from
+ the reference file. If `None`, all columns other than `join_key` are
+ returned.
+ file_format (Optional[str]): Explicit file format (`"parquet"`, `"csv"`,
+ `"json"`, or `"jsonl"`). If omitted, inferred from `path`.
+ format (Optional[str]): Alias for `file_format`.
+ delimiter (str): Field delimiter for CSV files. Defaults to `","`.
+ encoding (str): Text encoding for CSV/JSON/JSONL files. Defaults to
+ `"utf-8"`.
+ parse_csv_types (bool): Whether to infer primitive types (`int`, `float`,
+ `bool`, `str`) for CSV columns. Defaults to `True`.
+ exception_level (ExceptionLevel): Controls behavior when a key is not
+ found in the reference file. Defaults to `ExceptionLevel.QUIET`.
+ """
+ def __init__(
+ self,
+ path: str,
+ join_key: str = "",
+ *,
+ row_key: Optional[str] = None,
+ select_fields: Optional[list[str]] = None,
+ file_format: Optional[str] = None,
+ format: Optional[str] = None, # pylint: disable=redefined-builtin
+ delimiter: str = ',',
+ encoding: str = 'utf-8',
+ parse_csv_types: bool = True,
+ exception_level: ExceptionLevel = ExceptionLevel.QUIET,
+ ):
+ if not path:
+ raise ValueError("`path` must be provided for FileEnrichmentHandler.")
+ if not join_key and not row_key:
+ raise ValueError(
+ "Please specify `join_key` (or `row_key`) to identify the lookup "
+ "key column.")
+ self._path = path
+ self._join_key = join_key or (row_key or "")
+ self._row_key = row_key or self._join_key
+ self._select_fields = list(
+ select_fields) if select_fields is not None else None
+ raw_format = file_format or format or self._infer_format(path)
+ normalized_format = _FORMAT_ALIASES.get(raw_format.lower())
+ if not normalized_format:
+ raise ValueError(
+ f"Unsupported file format '{raw_format}'. Supported formats are: "
+ "parquet, csv, json, jsonl.")
+ self._format = normalized_format
+ self._delimiter = '\t' if (
+ self._format == 'tsv' and delimiter == ',') else delimiter
+ self._encoding = encoding
+ self._parse_csv_types = parse_csv_types
+ self._exception_level = exception_level
+ self._shared_handle = shared.Shared()
+ self._table: Optional[_LookupTable] = None
+ self._cached_schema: Optional[list[tuple[str, Any]]] = None
+
+ @staticmethod
+ def _infer_format(path: str) -> str:
+ clean_path = path.rstrip('*')
+ _, ext = os.path.splitext(clean_path)
+ ext = ext.lstrip('.').lower()
+ if ext in ('gz', 'bz2', 'zst', 'snappy'):
+ _, ext = os.path.splitext(clean_path[:-len(ext) - 1])
+ ext = ext.lstrip('.').lower()
+ if not ext:
+ raise ValueError(
+ f"Could not infer file format from path '{path}'. Please specify "
+ "`file_format` explicitly.")
+ return ext
+
+ def _resolve_paths(self) -> list[str]:
+ try:
+ match_results = FileSystems.match([self._path])
+ if match_results and match_results[0].metadata_list:
+ return [m.path for m in match_results[0].metadata_list]
+ except Exception:
+ pass
+ if FileSystems.exists(self._path):
+ return [self._path]
+ raise FileNotFoundError(f"Reference file not found at path: {self._path}")
+
+ def _determine_target_columns(self, all_columns: list[str]) -> list[str]:
+ if self._join_key not in all_columns:
+ raise KeyError(
+ f"join_key '{self._join_key}' not found in reference file columns: "
+ f"{all_columns}")
+ if self._select_fields is not None:
+ missing = [c for c in self._select_fields if c not in all_columns]
+ if missing:
+ raise KeyError(
+ f"select_fields {missing} not found in reference file columns: "
+ f"{all_columns}")
+ return list(self._select_fields)
+ return [c for c in all_columns if c != self._join_key]
+
+ def _load_parquet(self, file_paths: list[str]) -> _LookupTable:
+ if pa is None or pq is None:
+ raise ImportError(
+ "pyarrow is required to read Parquet files in
FileEnrichmentHandler.")
+ tables = []
+ for file_path in file_paths:
+ with FileSystems.open(file_path, 'rb') as f:
+ data = f.read()
+ tables.append(pq.read_table(pa.BufferReader(data)))
+
+ combined = pa.concat_tables(tables) if len(tables) > 1 else tables[0]
+ target_cols = self._determine_target_columns(combined.column_names)
+
+ rows = combined.to_pylist()
+ records: dict[Any, dict[str, Any]] = {}
+ str_records: dict[str, dict[str, Any]] = {}
+ col_values: dict[str, list[Any]] = {c: [] for c in target_cols}
+
+ for row in rows:
+ key = row.get(self._join_key)
+ projected = {c: row.get(c) for c in target_cols}
+ for c in target_cols:
+ col_values[c].append(projected[c])
+ records[key] = projected
+ if key is not None:
+ str_records[str(key)] = projected
+
+ schema: list[tuple[str, Any]] = []
+ arrow_schema = combined.schema
+ for c in target_cols:
+ field = arrow_schema.field(c)
+ py_type = _arrow_type_to_python_type(field.type)
+ has_null = any(v is None for v in col_values[c])
+ if has_null and py_type is not Any:
+ py_type = Optional[py_type]
+ schema.append((c, py_type))
+
+ return _LookupTable(records, str_records, schema)
+
+ def _load_csv(self, file_paths: list[str]) -> _LookupTable:
+ raw_rows: list[dict[str, Optional[str]]] = []
+ fieldnames: list[str] = []
+ for file_path in file_paths:
+ with FileSystems.open(file_path, 'rb') as f:
+ text = f.read().decode(self._encoding)
+ reader = csv.DictReader(io.StringIO(text), delimiter=self._delimiter)
+ if reader.fieldnames and not fieldnames:
+ fieldnames = list(reader.fieldnames)
+ for row in reader:
+ raw_rows.append(row)
+
+ target_cols = self._determine_target_columns(fieldnames)
+ key_raw = [r.get(self._join_key) for r in raw_rows]
+ typed_keys, _ = _infer_and_convert_csv_column(
+ key_raw, parse_types=self._parse_csv_types)
+
+ typed_cols: dict[str, list[Any]] = {}
+ schema: list[tuple[str, Any]] = []
+ for c in target_cols:
+ col_raw = [r.get(c) for r in raw_rows]
+ converted, col_type = _infer_and_convert_csv_column(
+ col_raw, parse_types=self._parse_csv_types)
+ typed_cols[c] = converted
+ schema.append((c, col_type))
+
+ records: dict[Any, dict[str, Any]] = {}
+ str_records: dict[str, dict[str, Any]] = {}
+ for idx, key in enumerate(typed_keys):
+ projected = {c: typed_cols[c][idx] for c in target_cols}
+ records[key] = projected
+ if key is not None:
+ str_records[str(key)] = projected
+ raw_k = key_raw[idx]
+ if raw_k is not None:
+ str_records[raw_k] = projected
+
+ return _LookupTable(records, str_records, schema)
+
+ def _load_json(self, file_paths: list[str]) -> _LookupTable:
+ raw_rows: list[dict[str, Any]] = []
+ all_columns: list[str] = []
+ seen_columns: set[str] = set()
+
+ for file_path in file_paths:
+ with FileSystems.open(file_path, 'rb') as f:
+ text = f.read().decode(self._encoding).strip()
+ if not text:
+ continue
+ if self._format == 'json' and text.startswith('['):
+ parsed = json.loads(text)
+ if not isinstance(parsed, list):
+ raise ValueError(
+ f"Expected JSON array of objects in '{file_path}', got "
+ f"{type(parsed).__name__}.")
+ file_rows = parsed
+ else:
+ file_rows = [
+ json.loads(line) for line in text.splitlines() if line.strip()
+ ]
+ for row in file_rows:
+ if not isinstance(row, dict):
+ raise ValueError(
+ f"Expected JSON object rows in '{file_path}', got "
+ f"{type(row).__name__}.")
+ for k in row.keys():
+ if k not in seen_columns:
+ seen_columns.add(k)
+ all_columns.append(k)
+ raw_rows.append(row)
+
+ target_cols = self._determine_target_columns(all_columns)
+ records: dict[Any, dict[str, Any]] = {}
+ str_records: dict[str, dict[str, Any]] = {}
+ col_values: dict[str, list[Any]] = {c: [] for c in target_cols}
+
+ for row in raw_rows:
+ key = row.get(self._join_key)
+ projected = {c: row.get(c) for c in target_cols}
+ for c in target_cols:
+ col_values[c].append(projected[c])
+ records[key] = projected
+ if key is not None:
+ str_records[str(key)] = projected
+
+ schema: list[tuple[str,
+ Any]] = [(c, _infer_python_column_type(col_values[c]))
+ for c in target_cols]
+ return _LookupTable(records, str_records, schema)
+
+ def _load_lookup_table(self) -> _LookupTable:
+ file_paths = self._resolve_paths()
+ if self._format == 'parquet':
+ return self._load_parquet(file_paths)
+ if self._format in ('csv', 'tsv'):
+ return self._load_csv(file_paths)
+ if self._format in ('json', 'jsonl'):
+ return self._load_json(file_paths)
+ raise ValueError(f"Unsupported file format: {self._format}")
+
+ def get_output_schema(self) -> list[tuple[str, Any]]:
+ """Returns the static `(field_name, field_type)` schema of enriched
fields."""
+ if self._cached_schema is not None:
+ return list(self._cached_schema)
+ try:
Review Comment:
**🟡 Medium**: `get_output_schema()` is called at pipeline construction time
inside `Enrichment.infer_output_type()` and `Enrichment.expand()`. Loading the
entire reference dataset into memory on the runner/submission machine during
graph construction can be very slow or fail with OOM/auth errors if the
reference file is large or resides on a remote filesystem only accessible to
workers. For Parquet, you can inspect only the footer metadata via
`pq.read_schema(...)` (and for CSV, only the header/sample if needed, or defer
full table loading to `__enter__`), rather than materializing the full
`_LookupTable` during pipeline construction.
##########
sdks/python/apache_beam/yaml/yaml_enrichment.py:
##########
@@ -80,6 +89,11 @@ def enrichment_transform(
-
[VertexAIFeatureStoreEnrichmentHandler](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store.html#apache_beam.transforms.enrichment_handlers.vertex_ai_feature_store.VertexAIFeatureStoreEnrichmentHandler)
Review Comment:
**🟢 Low**: Please also list `'File'` (`FileEnrichmentHandler`) in the
`enrichment_handler` docstring and `handler_config` documentation above so that
Beam YAML documentation generation includes the new `File` handler option.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]