eeroel commented on code in PR #37868:
URL: https://github.com/apache/arrow/pull/37868#discussion_r1404003297
##########
python/pyarrow/tests/test_dataset.py:
##########
@@ -981,6 +981,59 @@ def test_make_fragment(multisourcefs):
assert row_group_fragment.row_groups == [0]
[email protected]
[email protected]
+def test_make_fragment_with_size(s3_example_simple):
+ table, path, fs, uri, host, port, access_key, secret_key =
s3_example_simple
+
+ file_format = ds.ParquetFileFormat()
+ paths = [path]
+
+ fragments = [file_format.make_fragment(path, fs)
+ for path in paths]
+ dataset = ds.FileSystemDataset(
+ fragments, format=file_format, schema=table.schema, filesystem=fs
+ )
+
+ tbl = dataset.to_table()
+ assert tbl.equals(table)
+
+ # true sizes -> works
+ sizes_true = [dataset.filesystem.get_file_info(x).size for x in
dataset.files]
+ fragments_with_size = [file_format.make_fragment(path, fs, file_size=size)
+ for path, size in zip(paths, sizes_true)]
+ dataset_with_size = ds.FileSystemDataset(
+ fragments_with_size, format=file_format, schema=table.schema,
filesystem=fs
+ )
+ tbl = dataset.to_table()
+ assert tbl.equals(table)
+
+ # too small sizes -> error
+ sizes_toosmall = [1 for path in paths]
+ fragments_with_size = [file_format.make_fragment(path, fs, file_size=size)
+ for path, size in zip(paths, sizes_toosmall)]
+
+ dataset_with_size = ds.FileSystemDataset(
+ fragments_with_size, format=file_format, schema=table.schema,
filesystem=fs
+ )
+
+ with pytest.raises(pyarrow.lib.ArrowInvalid, match='Parquet file size is 1
bytes'):
+ table = dataset_with_size.to_table()
+
+ # too large sizes -> error
+ sizes_toolarge = [1000000 for path in paths]
+ fragments_with_size = [file_format.make_fragment(path, fs, file_size=size)
+ for path, size in zip(paths, sizes_toolarge)]
+
+ dataset_with_size = ds.FileSystemDataset(
+ fragments_with_size, format=file_format, schema=table.schema,
filesystem=fs
+ )
+
+ # invalid range
+ with pytest.raises(OSError, match='HTTP status 416'):
Review Comment:
Yeah maybe it's not ideal but at least "range not satisfiable" is the
expected outcome, instead of some error upstream for example? Since probably
there's no way to validate the size before making the request
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
