mapleFU commented on code in PR #49377:
URL: https://github.com/apache/arrow/pull/49377#discussion_r2981617188
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties]
_create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv,
"fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be
positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
+ if "fpp" in _bloom_opts:
+ fpp = _bloom_opts["fpp"]
+ if isinstance(fpp, float):
+ if fpp <= 0.0 or fpp >= 1.0:
+ raise ValueError(
+ f"'fpp' for column '{column}' must be in
(0.0, 1,0), got {fpp}")
Review Comment:
1,0 typo here?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties]
_create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv,
"fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
Review Comment:
ndv <= 0?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties]
_create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv,
"fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be
positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
+ if "fpp" in _bloom_opts:
+ fpp = _bloom_opts["fpp"]
+ if isinstance(fpp, float):
+ if fpp <= 0.0 or fpp >= 1.0:
+ raise ValueError(
+ f"'fpp' for column '{column}' must be in
(0.0, 1,0), got {fpp}")
+ bloom_opts.fpp = fpp
+ else:
+ raise TypeError(
+ f"'fpp' for column '{column}' must be a float")
+ elif isinstance(_bloom_opts, bool):
+ if not _bloom_opts:
Review Comment:
Add comments for `_bloom_opts == True`?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties]
_create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv,
"fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be
positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
Review Comment:
should we notice they're in bloom filter options?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
