pitrou commented on code in PR #13033:
URL: https://github.com/apache/arrow/pull/13033#discussion_r863685450
##########
python/pyarrow/tests/test_dataset.py:
##########
@@ -3058,25 +3108,52 @@ def test_feather_format(tempdir, dataset_reader):
dataset_reader.to_table(ds.dataset(basedir, format="feather"))
-def _create_parquet_dataset_simple(root_path):
+def _write_metadata_filesystem(
+ schema, path, filesystem, metadata_collector=None, **kwargs
+):
"""
- Creates a simple (flat files, no nested partitioning) Parquet dataset
+ Version of pq.write_metadata that works with a filesystem
"""
+ with filesystem.open_output_stream(path) as sink:
+ writer = pq.ParquetWriter(sink, schema, **kwargs)
+ writer.close()
+ if metadata_collector is not None:
+ # ParquetWriter doesn't expose the metadata until it's written. Write
+ # it and read it again.
+ with filesystem.open_input_file(path) as source:
+ metadata = pq.read_metadata(source)
+ for m in metadata_collector:
+ metadata.append_row_groups(m)
+ with filesystem.open_output_stream(path) as sink:
+ metadata.write_metadata_file(sink)
+
+
+def _create_parquet_dataset_simple(root_path, filesystem=None):
+ """
+ Creates a simple (flat files, no nested partitioning) Parquet dataset
+ """
metadata_collector = []
for i in range(4):
table = pa.table({'f1': [i] * 10, 'f2': np.random.randn(10)})
pq.write_to_dataset(
- table, str(root_path), metadata_collector=metadata_collector
+ table, str(root_path), filesystem=filesystem,
+ metadata_collector=metadata_collector
)
- metadata_path = str(root_path / '_metadata')
+ metadata_path = str(root_path) + '/_metadata'
Review Comment:
Ah, no, if it's an abstract path then it needs `/` indeed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
