raulcd commented on code in PR #49377:
URL: https://github.com/apache/arrow/pull/49377#discussion_r2845493242
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties]
_create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv,
"fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be
positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
+ if "fpp" in _bloom_opts:
+ fpp = _bloom_opts["fpp"]
+ if isinstance(fpp, float):
+ if fpp <= 0.0 or fpp >= 1.0:
+ raise ValueError(
+ f"'fpp' for column '{column}' must be in
(0.0, 1,0), got {fpp}")
+ bloom_opts.fpp = fpp
+ else:
+ raise TypeError(
+ f"'fpp' for column '{column}' must be a float")
+ elif isinstance(_bloom_opts, bool):
+ # if false do nothing, if true then just pass defaults
+ if not _bloom_opts:
+ continue
Review Comment:
I understand that bloom filter is disabled by default but should we be
explicit here calling `props.disable_bloom_filter(tobytes(column))`?
Otherwise exposing `Builder* disable_bloom_filter(const c_string& path)` on
`libparquet.pxd` seems slightly unnecessary as is never being used.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
