dmitry-chirkov-dremio commented on code in PR #53010:
URL: https://github.com/apache/arrow/pull/53010#discussion_r4346318843
##########
python/pyarrow/tests/parquet/test_parquet_file.py:
##########
@@ -201,6 +201,81 @@ def test_read_column_invalid_index():
f.reader.read_column(index)
+def test_dotted_top_level_column_takes_precedence():
+ table = pa.table({
+ 'a.b': [10, 20],
+ 'a': pa.array([{'b': 1}, {'b': 2}]),
+ 'other': [3, 4],
+ })
+ sink = pa.BufferOutputStream()
+ pq.write_table(table, sink)
+ data = sink.getvalue()
+ file_ = pq.ParquetFile(data)
+ expected = table.select(['a.b'])
+
+ assert file_.read(columns=['a.b']).equals(expected)
+ assert file_.read_row_group(0, columns=['a.b']).equals(expected)
+ assert pa.Table.from_batches(
+ list(file_.iter_batches(columns=['a.b']))).equals(expected)
+ assert pq.read_table(
+ pa.BufferReader(data), columns=['a.b']).equals(expected)
+ assert file_.read(columns=['a.b', 'a.b']).equals(expected)
+ assert file_.read(columns=['a.b', 'a']).equals(
+ table.select(['a.b', 'a']))
+
+
+def test_nested_column_selection_without_collision():
+ table = pa.table({
+ 'a': pa.array([
+ {'b': {'c': 1, 'd': 2}, 'x': 3},
+ {'b': {'c': 4, 'd': 5}, 'x': 6},
+ ]),
+ 'other': [5, 6],
+ })
+ sink = pa.BufferOutputStream()
+ pq.write_table(table, sink)
+ file_ = pq.ParquetFile(sink.getvalue())
+
+ assert file_.read(columns=['a']).equals(table.select(['a']))
+ assert file_.read(columns=['a.b']).to_pydict() == {
+ 'a': [{'b': {'c': 1, 'd': 2}}, {'b': {'c': 4, 'd': 5}}],
+ }
+ assert file_.read(columns=['a.b.c']).to_pydict() == {
+ 'a': [{'b': {'c': 1}}, {'b': {'c': 4}}],
+ }
+
+
+def test_non_conflicting_dotted_column_names():
+ table = pa.table({
+ 'metric.value': [10, 20],
+ 'a': pa.array([{'b': 1}, {'b': 2}]),
+ })
+ sink = pa.BufferOutputStream()
+ pq.write_table(table, sink)
+ file_ = pq.ParquetFile(sink.getvalue())
+
+ assert file_.read(columns=['metric.value']).equals(
+ table.select(['metric.value']))
+ assert file_.read(columns=['a.b']).to_pydict() == {
+ 'a': [{'b': 1}, {'b': 2}],
+ }
+
+
[email protected]
+def test_dotted_top_level_column_with_pandas_metadata():
Review Comment:
Is there a reason for this PR to be solely focused on top-level collisions?
```
table = pa.table({
"a.b": pa.array([{"c": 1}]),
"a": pa.array([{"b": {"c": 2}}]),
})
```
a.b.c will still return both columns here, no?
##########
python/pyarrow/tests/parquet/test_parquet_file.py:
##########
@@ -201,6 +201,81 @@ def test_read_column_invalid_index():
f.reader.read_column(index)
+def test_dotted_top_level_column_takes_precedence():
Review Comment:
Could we parameterize this test (or add couple of test cases) with both
schema orders: the literal "a.b" before and after struct "a"? That would
capture that exact-name precedence does not depend on field order (and capture
order preservation for nested cases where all fields are returned)
##########
python/pyarrow/tests/parquet/test_parquet_file.py:
##########
@@ -201,6 +201,81 @@ def test_read_column_invalid_index():
f.reader.read_column(index)
+def test_dotted_top_level_column_takes_precedence():
Review Comment:
3 returned columns, order maintained
```
schema = pa.schema([
pa.field("a.b.c", pa.struct([
pa.field("d", pa.int64()),
])),
pa.field("a.b", pa.struct([
pa.field("c", pa.struct([
pa.field("d", pa.int64()),
])),
])),
pa.field("a", pa.struct([
pa.field("b", pa.struct([
pa.field("c", pa.struct([
pa.field("d", pa.int64()),
])),
])),
])),
])
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
