leaves12138 commented on code in PR #10009:
URL: https://github.com/apache/paimon/pull/10009#discussion_r4056693421
##########
paimon-python/pypaimon/tests/native_plan_integration_test.py:
##########
@@ -262,11 +264,312 @@ def test_native_read_primary_key_matches_python(self):
with patch(
'pypaimon.read.table_read.TableRead._create_split_read',
side_effect=AssertionError('Python reader was used')):
+ native_read_instance = builder.new_read()
native = sorted(
- builder.new_read().to_arrow(plan.splits()).to_pylist(),
+ native_read_instance.to_arrow(plan.splits()).to_pylist(),
key=lambda row: row['k'])
+ native_read_instance.include_row_kind = True
+ with_row_kind = native_read_instance.to_arrow(
+ plan.splits()).to_pylist()
self.assertEqual(native, normal)
+ self.assertEqual(
+ {row['_row_kind'] for row in with_row_kind}, {'+I'})
+ self.assertEqual(
+ sorted(({key: value for key, value in row.items()
+ if key != '_row_kind'} for row in with_row_kind),
+ key=lambda row: row['k']),
+ normal,
+ )
+
+ @unittest.skipUnless(
+ native_method_available('ReadBuilder', 'with_nested_projection'),
+ "pypaimon-rust nested native reader API not installed")
+ def test_native_read_nested_rows_and_map_keys_across_formats(self):
+ base_fields = [
+ ('id', pa.int64()),
+ ('payload', pa.struct([
+ ('details', pa.struct([
+ ('score', pa.int32()),
+ ('label', pa.string()),
+ ])),
+ ('ignored', pa.string()),
+ ])),
+ ]
+ rows = [
+ {'id': 1,
+ 'payload': {'details': {'score': 7, 'label': 'a'}, 'ignored':
'x'},
+ 'attrs': {'selected': 10, 'other': 11},
+ 'top.with.dot': 'first'},
+ {'id': 2, 'payload': None, 'attrs': None, 'top.with.dot':
'second'},
+ {'id': 3,
+ 'payload': {'details': None, 'ignored': 'z'},
+ 'attrs': {},
+ 'top.with.dot': 'third'},
+ ]
+ formats = ['parquet', 'orc', 'avro', 'row']
+ if importlib.util.find_spec('mosaic') is not None:
Review Comment:
[P2] Gate the Mosaic subcase on nested-ROW support, not just importability
`paimon-mosaic==0.1.0` satisfies the existing optional dependency
(`paimon-mosaic>=0.1.0` in `setup.py`) and makes this condition true, but its
writer cannot write the nested `payload` Struct used by this test. With that
supported dependency version and the freshly built #887 extension, this test
fails while creating the fixture, before it reaches either reader:
`RuntimeError: writer open failed: unsupported DataType: Struct([Field { name:
"details", data_type: Struct(...) }, ...])`. The other 171 tests in the
targeted suites passed.
Please gate this subcase on a Mosaic version/capability that supports the
schema, adapt its fixture to supported types, or explicitly raise the optional
dependency minimum if requiring a newer writer is intentional. Otherwise
installing an allowed Mosaic version makes the new native-read suite fail for
reasons unrelated to the reader being tested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
