github-actions[bot] commented on code in PR #41452:
URL: https://github.com/apache/beam/pull/41452#discussion_r4617603217


##########
sdks/python/apache_beam/transforms/enrichment_handlers/file.py:
##########
@@ -0,0 +1,566 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import csv
+import io
+import itertools
+import json
+import logging
+import math
+import os
+from typing import Any
+from typing import Optional
+
+import apache_beam as beam
+from apache_beam.io.filesystems import FileSystems
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
+from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel
+from apache_beam.utils import shared
+
+try:
+  import pyarrow as pa
+  import pyarrow.parquet as pq
+except ImportError:
+  pa = None  # type: ignore[assignment]
+  pq = None  # type: ignore[assignment]
+
+__all__ = [
+    'FileEnrichmentHandler',
+]
+
+_LOGGER = logging.getLogger(__name__)
+
+_FORMAT_ALIASES = {
+    'parquet': 'parquet',
+    'pq': 'parquet',
+    'csv': 'csv',
+    'tsv': 'tsv',
+    'json': 'json',
+    'jsonl': 'jsonl',
+    'ndjson': 'jsonl',
+}
+
+
+class _LookupTable:
+  """Weak-referenceable container for in-memory reference records and 
schema."""
+  def __init__(
+      self,
+      records: dict[Any, dict[str, Any]],
+      str_records: dict[str, dict[str, Any]],
+      schema: list[tuple[str, Any]],
+  ):
+    self.records = records
+    self.str_records = str_records
+    self.schema = schema
+
+  def get(self, key: Any) -> Optional[dict[str, Any]]:
+    if key is None:
+      return self.records.get(None)

Review Comment:
   **🟠 High**: `_load_parquet`, `_load_csv`, and `_load_json` insert 
`records[None] = projected` whenever a reference row has a null or empty 
`join_key`. Returning `self.records.get(None)` when the input row's lookup `key 
is None` causes null foreign keys in the input PCollection to falsely match 
those null-keyed reference rows instead of being treated as lookup misses. A 
`None` lookup key should always be a miss (and reference rows with `key is 
None` should not be indexed).
   
   ```suggestion
         return None
   ```



##########
sdks/python/apache_beam/transforms/enrichment.py:
##########
@@ -55,32 +64,143 @@ def has_valid_redis_address(host: str, port: int) -> bool:
   return False
 
 
-def cross_join(left: dict[str, Any], right: dict[str, Any]) -> beam.Row:
+def cross_join(
+    left: dict[str, Any],
+    right: dict[str, Any],
+    prefix: Optional[str] = None) -> beam.Row:
   """performs a cross join on two `dict` objects.
 
     Joins the columns of the right row onto the left row.
 
     Args:
       left (dict[str, Any]): input request dictionary.
       right (dict[str, Any]): response dictionary from the API.
+      prefix (Optional[str]): optional prefix to prepend to keys from `right`
+        to avoid column name collisions.
 
     Returns:
       `beam.Row` containing the merged columns.
   """
   for k, v in right.items():
-    if k not in left:
+    target_key = f"{prefix}{k}" if prefix else k
+    if target_key not in left:
       # Don't override the values in left.
-      left[k] = v
-    elif left[k] != v:
+      left[target_key] = v
+    elif left[target_key] != v:
       _LOGGER.warning(
           '%s exists in the input row as well the row fetched '
           'from API but have different values - %s and %s. Using the input '
           'value (%s) for the enriched row. You can override this behavior by '
           'passing a custom `join_fn` to Enrichment transform.' %
-          (k, left[k], v, left[k]))
+          (target_key, left[target_key], v, left[target_key]))
   return beam.Row(**left)
 
 
+class _EnrichmentJoinFn(beam.DoFn):
+  """DoFn that joins `(request, response)` tuples with `on_miss` and 
`prefix`."""
+  def __init__(
+      self,
+      join_fn: JoinFn,
+      use_custom_types: bool = False,
+      on_miss: str = 'null',
+      prefix: Optional[str] = None,
+      null_fields: Optional[list[str]] = None,
+      source_handler: Optional[Any] = None,
+      dead_letter_tag: Optional[str] = None,
+  ):
+    self._join_fn = join_fn
+    self._use_custom_types = use_custom_types
+    self._on_miss = on_miss
+    self._prefix = prefix
+    self._null_fields = list(null_fields) if null_fields else []
+    self._source_handler = source_handler
+    self._dead_letter_tag = dead_letter_tag
+
+  def setup(self):
+    if (not self._null_fields and self._source_handler is not None and
+        hasattr(self._source_handler, 'get_output_schema')):
+      try:
+        self._null_fields = [
+            name for name, _ in self._source_handler.get_output_schema()
+        ]
+      except Exception:
+        pass
+
+  @staticmethod
+  def _is_miss(response: Any) -> bool:
+    if response is None:
+      return True
+    if hasattr(response, '_asdict'):
+      return len(response._asdict()) == 0
+    if isinstance(response, Mapping):
+      return len(response) == 0
+    return False
+
+  @staticmethod
+  def _to_dict(obj: Any) -> dict[str, Any]:
+    if hasattr(obj, '_asdict'):
+      return dict(obj._asdict())
+    return dict(obj)
+
+  def process(self, element: tuple[Any, Any], *args, **kwargs):
+    request, response = element
+    if self._is_miss(response):
+      if self._on_miss == 'drop':
+        return
+      if self._on_miss == 'fail':
+        raise KeyError(f"Enrichment lookup missed for element: {request}")
+      if self._on_miss == 'dlq':
+        try:
+          raise KeyError(f"Enrichment lookup missed for element: {request}")
+        except Exception as exn:
+          if self._dead_letter_tag:
+            yield pvalue.TaggedOutput(
+                self._dead_letter_tag,
+                (
+                    request,
+                    (
+                        type(exn),
+                        repr(exn),
+                        traceback.format_exception(*sys.exc_info()))))
+            return
+          raise
+
+      # on_miss == 'null' (left join)
+      if self._use_custom_types:
+        yield self._join_fn(request, response)
+        return
+      left_dict = self._to_dict(request)
+      right_dict: dict[str, Any] = {f: None for f in self._null_fields}
+      if self._prefix:
+        right_dict = {f"{self._prefix}{k}": v for k, v in right_dict.items()}
+      yield self._join_fn(left_dict, right_dict)

Review Comment:
   **🟡 Medium**: In `_EnrichmentJoinFn.process`, the `on_miss == 'null'` branch 
invokes `self._join_fn(...)` outside the `try ... except Exception as exn:` 
block below (lines 179-200). If a custom `join_fn` raises an exception when 
handling a miss with `on_miss='null'` and `with_exception_handling(...)` is 
enabled, the exception will fail the pipeline instead of being routed to 
`self._dead_letter_tag`.



##########
sdks/python/apache_beam/transforms/enrichment_handlers/file.py:
##########
@@ -0,0 +1,566 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import csv
+import io
+import itertools
+import json
+import logging
+import math
+import os
+from typing import Any
+from typing import Optional
+
+import apache_beam as beam
+from apache_beam.io.filesystems import FileSystems
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
+from apache_beam.transforms.enrichment_handlers.utils import ExceptionLevel
+from apache_beam.utils import shared
+
+try:
+  import pyarrow as pa
+  import pyarrow.parquet as pq
+except ImportError:
+  pa = None  # type: ignore[assignment]
+  pq = None  # type: ignore[assignment]
+
+__all__ = [
+    'FileEnrichmentHandler',
+]
+
+_LOGGER = logging.getLogger(__name__)
+
+_FORMAT_ALIASES = {
+    'parquet': 'parquet',
+    'pq': 'parquet',
+    'csv': 'csv',
+    'tsv': 'tsv',
+    'json': 'json',
+    'jsonl': 'jsonl',
+    'ndjson': 'jsonl',
+}
+
+
+class _LookupTable:
+  """Weak-referenceable container for in-memory reference records and 
schema."""
+  def __init__(
+      self,
+      records: dict[Any, dict[str, Any]],
+      str_records: dict[str, dict[str, Any]],
+      schema: list[tuple[str, Any]],
+  ):
+    self.records = records
+    self.str_records = str_records
+    self.schema = schema
+
+  def get(self, key: Any) -> Optional[dict[str, Any]]:
+    if key is None:
+      return self.records.get(None)
+    if key in self.records:
+      return self.records[key]
+    str_key = str(key)
+    if str_key in self.str_records:
+      return self.str_records[str_key]
+    return None
+
+
+def _arrow_type_to_python_type(arrow_type: Any) -> Any:
+  if pa is None:
+    return Any
+  if pa.types.is_boolean(arrow_type):
+    return bool
+  if pa.types.is_integer(arrow_type):
+    return int
+  if pa.types.is_floating(arrow_type) or pa.types.is_decimal(arrow_type):
+    return float
+  if (pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type) or
+      pa.types.is_unicode(arrow_type)):
+    return str
+  if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type):
+    return bytes
+  if pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type):
+    elem_type = _arrow_type_to_python_type(arrow_type.value_type)
+    return list[elem_type]  # type: ignore[valid-type]
+  return Any
+
+
+def _is_int_string(val: str) -> bool:
+  s = val.strip()
+  if not s:
+    return False
+  unsigned = s[1:] if s[0] in ('+', '-') else s
+  if not unsigned.isdigit():
+    return False
+  if len(unsigned) > 1 and unsigned.startswith('0'):
+    return False
+  return True
+
+
+def _is_float_string(val: str) -> bool:
+  s = val.strip()
+  if not s:
+    return False
+  try:
+    parsed = float(s)
+    return not (math.isnan(parsed) or math.isinf(parsed))
+  except ValueError:
+    return False
+
+
+def _infer_and_convert_csv_column(
+    raw_values: list[Optional[str]],
+    parse_types: bool = True) -> tuple[list[Any], Any]:
+  has_null = any(v is None or v == '' for v in raw_values)
+  non_empty = [v for v in raw_values if v is not None and v != '']
+  if not non_empty or not parse_types:
+    converted = [None if (v is None or v == '') else v for v in raw_values]
+    base_type: Any = str
+    return converted, (Optional[base_type] if has_null else base_type)
+
+  if all(v.strip().lower() in ('true', 'false') for v in non_empty):
+    converted = [
+        None if (v is None or v == '') else (v.strip().lower() == 'true')
+        for v in raw_values
+    ]
+    base_type = bool
+  elif all(_is_int_string(v) for v in non_empty):
+    converted = [
+        None if (v is None or v == '') else int(v.strip()) for v in raw_values
+    ]
+    base_type = int
+  elif all(_is_float_string(v) for v in non_empty):
+    converted = [
+        None if (v is None or v == '') else float(v.strip()) for v in 
raw_values
+    ]
+    base_type = float
+  else:
+    converted = [None if (v is None or v == '') else v for v in raw_values]
+    base_type = str
+
+  return converted, (Optional[base_type] if has_null else base_type)
+
+
+def _infer_python_column_type(values: list[Any]) -> Any:
+  has_null = any(v is None for v in values)
+  non_null = [v for v in values if v is not None]
+  if not non_null:
+    return Optional[Any]
+
+  types_seen = {type(v) for v in non_null}
+  if types_seen == {bool}:
+    base_type: Any = bool
+  elif types_seen == {int}:
+    base_type = int
+  elif types_seen <= {int, float}:
+    base_type = float
+  elif types_seen == {str}:
+    base_type = str
+  elif types_seen == {bytes}:
+    base_type = bytes
+  else:
+    base_type = Any
+
+  return Optional[base_type] if has_null else base_type
+
+
+class FileEnrichmentHandler(EnrichmentSourceHandler[beam.Row, beam.Row]):
+  """Enrichment handler for file-backed reference tables (Parquet, CSV, 
JSON/JSONL).
+
+  Loads a reference file via :class:`apache_beam.io.filesystems.FileSystems`
+  in worker ``__enter__`` and shares the in-memory lookup dictionary across
+  worker threads via :class:`apache_beam.utils.shared.Shared`.
+
+  Args:
+    path (str): Path or file pattern to the reference file (e.g. `.parquet`,
+      `.csv`, `.json`, or `.jsonl` on local disk, GCS, or S3).
+    join_key (str): Key column name in the reference file (and in the input
+      `beam.Row` unless `row_key` is specified separately).
+    row_key (Optional[str]): Field name in the input `beam.Row` to use for
+      lookup. Defaults to `join_key`.
+    select_fields (Optional[list[str]]): List of column names to select from
+      the reference file. If `None`, all columns other than `join_key` are
+      returned.
+    file_format (Optional[str]): Explicit file format (`"parquet"`, `"csv"`,
+      `"json"`, or `"jsonl"`). If omitted, inferred from `path`.
+    format (Optional[str]): Alias for `file_format`.
+    delimiter (str): Field delimiter for CSV files. Defaults to `","`.
+    encoding (str): Text encoding for CSV/JSON/JSONL files. Defaults to
+      `"utf-8"`.
+    parse_csv_types (bool): Whether to infer primitive types (`int`, `float`,
+      `bool`, `str`) for CSV columns. Defaults to `True`.
+    exception_level (ExceptionLevel): Controls behavior when a key is not
+      found in the reference file. Defaults to `ExceptionLevel.QUIET`.
+  """
+  def __init__(
+      self,
+      path: str,
+      join_key: str = "",
+      *,
+      row_key: Optional[str] = None,
+      select_fields: Optional[list[str]] = None,
+      file_format: Optional[str] = None,
+      format: Optional[str] = None,  # pylint: disable=redefined-builtin
+      delimiter: str = ',',
+      encoding: str = 'utf-8',
+      parse_csv_types: bool = True,
+      exception_level: ExceptionLevel = ExceptionLevel.QUIET,
+  ):
+    if not path:
+      raise ValueError("`path` must be provided for FileEnrichmentHandler.")
+    if not join_key and not row_key:
+      raise ValueError(
+          "Please specify `join_key` (or `row_key`) to identify the lookup "
+          "key column.")
+    self._path = path
+    self._join_key = join_key or (row_key or "")
+    self._row_key = row_key or self._join_key
+    self._select_fields = list(
+        select_fields) if select_fields is not None else None
+    raw_format = file_format or format or self._infer_format(path)
+    normalized_format = _FORMAT_ALIASES.get(raw_format.lower())
+    if not normalized_format:
+      raise ValueError(
+          f"Unsupported file format '{raw_format}'. Supported formats are: "
+          "parquet, csv, json, jsonl.")
+    self._format = normalized_format
+    self._delimiter = '\t' if (
+        self._format == 'tsv' and delimiter == ',') else delimiter
+    self._encoding = encoding
+    self._parse_csv_types = parse_csv_types
+    self._exception_level = exception_level
+    self._shared_handle = shared.Shared()
+    self._table: Optional[_LookupTable] = None
+    self._cached_schema: Optional[list[tuple[str, Any]]] = None
+
+  @staticmethod
+  def _infer_format(path: str) -> str:
+    clean_path = path.rstrip('*')
+    _, ext = os.path.splitext(clean_path)
+    ext = ext.lstrip('.').lower()
+    if ext in ('gz', 'bz2', 'zst', 'snappy'):
+      _, ext = os.path.splitext(clean_path[:-len(ext) - 1])
+      ext = ext.lstrip('.').lower()
+    if not ext:
+      raise ValueError(
+          f"Could not infer file format from path '{path}'. Please specify "
+          "`file_format` explicitly.")
+    return ext
+
+  def _resolve_paths(self) -> list[str]:
+    try:
+      match_results = FileSystems.match([self._path])
+      if match_results and match_results[0].metadata_list:
+        return [m.path for m in match_results[0].metadata_list]
+    except Exception:
+      pass
+    if FileSystems.exists(self._path):
+      return [self._path]
+    raise FileNotFoundError(f"Reference file not found at path: {self._path}")
+
+  def _determine_target_columns(self, all_columns: list[str]) -> list[str]:
+    if self._join_key not in all_columns:
+      raise KeyError(
+          f"join_key '{self._join_key}' not found in reference file columns: "
+          f"{all_columns}")
+    if self._select_fields is not None:
+      missing = [c for c in self._select_fields if c not in all_columns]
+      if missing:
+        raise KeyError(
+            f"select_fields {missing} not found in reference file columns: "
+            f"{all_columns}")
+      return list(self._select_fields)
+    return [c for c in all_columns if c != self._join_key]
+
+  def _effective_text_encoding(self) -> str:
+    if self._encoding.lower().replace('_', '-') in ('utf-8', 'utf8'):
+      return 'utf-8-sig'
+    return self._encoding
+
+  def _load_parquet(self, file_paths: list[str]) -> _LookupTable:
+    if pa is None or pq is None:
+      raise ImportError(
+          "pyarrow is required to read Parquet files in 
FileEnrichmentHandler.")
+    tables = []
+    for file_path in file_paths:
+      with FileSystems.open(file_path, 'rb') as f:
+        tables.append(pq.read_table(f))
+
+    combined = pa.concat_tables(tables) if len(tables) > 1 else tables[0]
+    target_cols = self._determine_target_columns(combined.column_names)
+
+    rows = combined.to_pylist()
+    records: dict[Any, dict[str, Any]] = {}
+    str_records: dict[str, dict[str, Any]] = {}
+    col_values: dict[str, list[Any]] = {c: [] for c in target_cols}
+
+    for row in rows:
+      key = row.get(self._join_key)
+      projected = {c: row.get(c) for c in target_cols}
+      for c in target_cols:
+        col_values[c].append(projected[c])
+      records[key] = projected
+      if key is not None:
+        str_records[str(key)] = projected
+
+    schema: list[tuple[str, Any]] = []
+    arrow_schema = combined.schema
+    for c in target_cols:
+      field = arrow_schema.field(c)
+      py_type = _arrow_type_to_python_type(field.type)
+      has_null = any(v is None for v in col_values[c])
+      if has_null and py_type is not Any:
+        py_type = Optional[py_type]
+      schema.append((c, py_type))
+
+    return _LookupTable(records, str_records, schema)
+
+  def _load_csv(self, file_paths: list[str]) -> _LookupTable:
+    raw_rows: list[dict[str, Optional[str]]] = []
+    fieldnames: list[str] = []
+    seen_fieldnames: set[str] = set()
+    text_encoding = self._effective_text_encoding()
+    for file_path in file_paths:
+      with FileSystems.open(file_path, 'rb') as f:
+        text = f.read().decode(text_encoding)
+      if not text.strip():
+        continue
+      reader = csv.DictReader(io.StringIO(text), delimiter=self._delimiter)
+      if reader.fieldnames:
+        file_cols = list(reader.fieldnames)
+        self._determine_target_columns(file_cols)
+        for col in file_cols:
+          if col not in seen_fieldnames:
+            seen_fieldnames.add(col)
+            fieldnames.append(col)
+      for row in reader:
+        raw_rows.append(row)
+
+    target_cols = self._determine_target_columns(fieldnames)
+    key_raw = [r.get(self._join_key) for r in raw_rows]
+    typed_keys, _ = _infer_and_convert_csv_column(
+        key_raw, parse_types=self._parse_csv_types)
+
+    typed_cols: dict[str, list[Any]] = {}
+    schema: list[tuple[str, Any]] = []
+    for c in target_cols:
+      col_raw = [r.get(c) for r in raw_rows]
+      converted, col_type = _infer_and_convert_csv_column(
+          col_raw, parse_types=self._parse_csv_types)
+      typed_cols[c] = converted
+      schema.append((c, col_type))
+
+    records: dict[Any, dict[str, Any]] = {}
+    str_records: dict[str, dict[str, Any]] = {}
+    for idx, key in enumerate(typed_keys):
+      projected = {c: typed_cols[c][idx] for c in target_cols}
+      records[key] = projected
+      if key is not None:
+        str_records[str(key)] = projected
+      raw_k = key_raw[idx]
+      if raw_k is not None:
+        str_records[raw_k] = projected
+
+    return _LookupTable(records, str_records, schema)
+
+  @staticmethod
+  def _parse_json_text(text: str, file_path: str) -> list[dict[str, Any]]:
+    stripped = text.strip()
+    if not stripped:
+      return []
+    if stripped.startswith('['):
+      parsed = json.loads(stripped)
+      if not isinstance(parsed, list):
+        raise ValueError(
+            f"Expected JSON array of objects in '{file_path}', got "
+            f"{type(parsed).__name__}.")
+      return parsed
+    try:
+      parsed = json.loads(stripped)
+      if isinstance(parsed, list):
+        return parsed
+      if isinstance(parsed, dict):
+        return [parsed]
+      raise ValueError(
+          f"Expected JSON object or array of objects in '{file_path}', got "
+          f"{type(parsed).__name__}.")
+    except json.JSONDecodeError:
+      return [
+          json.loads(line) for line in stripped.splitlines() if line.strip()
+      ]
+
+  def _load_json(self, file_paths: list[str]) -> _LookupTable:
+    raw_rows: list[dict[str, Any]] = []
+    all_columns: list[str] = []
+    seen_columns: set[str] = set()
+    text_encoding = self._effective_text_encoding()
+
+    for file_path in file_paths:
+      with FileSystems.open(file_path, 'rb') as f:
+        text = f.read().decode(text_encoding)
+      file_rows = self._parse_json_text(text, file_path)
+      for row in file_rows:
+        if not isinstance(row, dict):
+          raise ValueError(
+              f"Expected JSON object rows in '{file_path}', got "
+              f"{type(row).__name__}.")
+        for k in row.keys():
+          if k not in seen_columns:
+            seen_columns.add(k)
+            all_columns.append(k)
+        raw_rows.append(row)
+
+    target_cols = self._determine_target_columns(all_columns)
+    records: dict[Any, dict[str, Any]] = {}
+    str_records: dict[str, dict[str, Any]] = {}
+    col_values: dict[str, list[Any]] = {c: [] for c in target_cols}
+
+    for row in raw_rows:
+      key = row.get(self._join_key)
+      projected = {c: row.get(c) for c in target_cols}
+      for c in target_cols:
+        col_values[c].append(projected[c])
+      records[key] = projected
+      if key is not None:
+        str_records[str(key)] = projected
+
+    schema: list[tuple[str,
+                       Any]] = [(c, _infer_python_column_type(col_values[c]))
+                                for c in target_cols]
+    return _LookupTable(records, str_records, schema)
+
+  def _load_lookup_table(self) -> _LookupTable:
+    file_paths = self._resolve_paths()
+    if self._format == 'parquet':
+      return self._load_parquet(file_paths)
+    if self._format in ('csv', 'tsv'):
+      return self._load_csv(file_paths)
+    if self._format in ('json', 'jsonl'):
+      return self._load_json(file_paths)
+    raise ValueError(f"Unsupported file format: {self._format}")
+
+  def _peek_schema(self) -> list[tuple[str, Any]]:
+    """Infers output schema from file metadata/sample without loading all 
rows."""
+    file_paths = self._resolve_paths()
+    first_path = file_paths[0]
+    text_encoding = self._effective_text_encoding()
+
+    if self._format == 'parquet':
+      if pa is None or pq is None:
+        raise ImportError(
+            "pyarrow is required to read Parquet files in "
+            "FileEnrichmentHandler.")
+      with FileSystems.open(first_path, 'rb') as f:
+        arrow_schema = pq.read_schema(f)
+      target_cols = self._determine_target_columns(list(arrow_schema.names))
+      return [(c, _arrow_type_to_python_type(arrow_schema.field(c).type))

Review Comment:
   **🟠 High**: `_peek_schema()` ignores field nullability on the Arrow schema 
(`arrow_schema.field(c).nullable`) and always returns non-`Optional` types for 
Parquet columns. Because `get_output_schema()` caches `_peek_schema()` during 
pipeline construction, using `Enrichment(handler, on_miss='drop')` (or 
`'fail'`/`'dlq'`) on a Parquet reference file with nullable columns (such as 
`'opt_val': [1.0, None]` in `test_schema_inference_json_jsonl_and_arrow_types`) 
sets a non-nullable `RowTypeConstraint` on the output PCollection, which fails 
when encoding `None` values at runtime. Check `arrow_schema.field(c).nullable` 
(or Parquet footer column chunk `null_count` statistics) and wrap in 
`Optional[...]` when nullable.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to