github-actions[bot] commented on code in PR #41453:
URL: https://github.com/apache/beam/pull/41453#discussion_r4619190169


##########
sdks/python/apache_beam/yaml/yaml_enrichment_test.py:
##########
@@ -15,18 +15,30 @@
 # limitations under the License.
 #
 
+import json

Review Comment:
   **🟢 Nit**: `import json` is unused in this test module and may be flagged by 
flake8/pylint.



##########
sdks/python/apache_beam/yaml/yaml_enrichment_test.py:
##########
@@ -72,6 +101,182 @@ def test_enrichment_with_bigquery(self):
                     ''')
         assert_that(result, equal_to(input_data))
 
+  def test_yaml_file_enrichment_parquet_dlq_and_prefix(self):
+    if pa is None or pq is None:
+      self.skipTest('pyarrow is not installed')
+    parquet_path = os.path.join(self.temp_dir.name, 'sku_catalog.parquet')
+    pq.write_table(
+        pa.table({
+            'sku_id': ['SKU-1', 'SKU-2'],
+            'category': ['electronics', 'books'],
+            'unit_cost': [99.5, 14.25],
+            'ignored_col': ['x', 'y'],
+        }),
+        parquet_path,
+    )
+
+    input_data = [
+        Row(sku_id='SKU-1', qty=2),
+        Row(sku_id='SKU-MISSING', qty=1),
+    ]
+
+    with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
+        ['--yaml_experimental_features=Enrichment'])) as p:
+      input_pcoll = p | 'CreateInput' >> beam.Create(input_data)
+      result = input_pcoll | YamlTransform(
+          f'''
+          type: composite
+          input:
+            events: input
+          transforms:
+            - type: Enrichment
+              name: EnrichEvents
+              input: events
+              config:
+                enrichment_handler: File
+                handler_config:
+                  path: '{parquet_path}'
+                  join_key: sku_id
+                  select_fields: [category, unit_cost]
+                prefix: sku_
+                on_miss: dlq
+                error_handling:
+                  output: failed_enrichment
+          output:
+            enriched: EnrichEvents
+            dlq: EnrichEvents.failed_enrichment
+          ''')
+      assert_that(
+          result['enriched'],
+          equal_to([
+              Row(
+                  sku_id='SKU-1',
+                  qty=2,
+                  sku_category='electronics',
+                  sku_unit_cost=99.5)
+          ]),
+          label='CheckEnriched',
+      )
+      assert_that(
+          result['dlq'] | beam.Map(lambda r: r.element),
+          equal_to([Row(sku_id='SKU-MISSING', qty=1)]),
+          label='CheckDLQ',
+      )
+
+  def test_yaml_file_enrichment_csv_on_miss_null_and_drop(self):
+    csv_path = os.path.join(self.temp_dir.name, 'catalog.csv')
+    with open(csv_path, 'w', encoding='utf-8') as f:
+      f.write('sku_id,category,unit_cost\nSKU-1,books,10.0\n')
+
+    input_data = [
+        Row(sku_id='SKU-1', qty=1),
+        Row(sku_id='SKU-2', qty=3),
+    ]
+
+    with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
+        ['--yaml_experimental_features=Enrichment'])) as p:
+      input_pcoll = p | 'CreateNullInput' >> beam.Create(input_data)
+      res_null = input_pcoll | 'EnrichNull' >> YamlTransform(
+          f'''
+          type: Enrichment
+          config:
+            enrichment_handler: File
+            handler_config:
+              path: '{csv_path}'
+              join_key: sku_id
+              select_fields: [category, unit_cost]
+            prefix: sku_
+            on_miss: null

Review Comment:
   **🟢 Nit**: In YAML, unquoted `null` is parsed as Python `None` rather than 
the string `'null'`. While `Enrichment` normalizes `None` to `'null'`, quoting 
it (`on_miss: "null"` or `on_miss: 'null'`) ensures the string value `'null'` 
is explicitly tested as documented in the transform docstring.
   
   ```suggestion
               on_miss: 'null'
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to