rok commented on code in PR #55665:
URL: https://github.com/apache/arrow/pull/55665#discussion_r4653262126
##########
cpp/src/arrow/dataset/dataset.cc:
##########
@@ -321,6 +322,129 @@ Result<FragmentIterator>
UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}
+FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
+ compute::Expression filter)
+ : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
+ fragment_(std::move(fragment)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
+ return fragment_->ReadPhysicalSchema();
+}
+
+Status FilteredFragment::ClearCachedMetadata() {
+ RETURN_NOT_OK(Fragment::ClearCachedMetadata());
+ return fragment_->ClearCachedMetadata();
+}
+
+std::string FilteredFragment::ToString() const {
+ // The scanner uses ToString() for the __filename augmented field, so this
must
+ // match the wrapped fragment.
+ return fragment_->ToString();
+}
+
+Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) {
+ const auto& dataset_schema = options->dataset_schema;
+ ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(filter,
+ SimplifyWithGuarantee(std::move(filter),
partition_expression_));
+ if (!filter.IsSatisfiable()) {
+ return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
+ }
+
+ // Combine any existing filter in the scan options with the fragment's
filter,
+ // so the wrapped fragment can skip data that cannot match.
+ auto fragment_options = std::make_shared<ScanOptions>(*options);
+ ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
+ compute::and_(options->filter,
filter).Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(auto batches,
fragment_->ScanBatchesAsync(fragment_options));
+
+ // The filter might simplify to true if it only references partition fields,
+ // in which case we can skip filtering batches.
+ if (filter == compute::literal(true)) {
+ return batches;
+ }
+
+ // Apply the fragment's filter to scanned rows.
+ // The filter in the scan options is only used for data pruning at this point
+ // and will be used to filter rows later, so don't duplicate that work.
+ auto guarantee = partition_expression_;
+ auto pool = options->pool;
+ return MakeMappedGenerator(
+ std::move(batches),
+ [filter, dataset_schema, guarantee, pool](const
std::shared_ptr<RecordBatch>& batch)
+ -> Result<std::shared_ptr<RecordBatch>> {
+ compute::ExecContext exec_context(pool);
+ ARROW_ASSIGN_OR_RAISE(auto exec_batch,
+ compute::MakeExecBatch(*dataset_schema, batch,
guarantee));
+ ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression(
+ filter, exec_batch,
&exec_context));
+ if (mask.is_scalar()) {
+ const auto& selected = mask.scalar_as<BooleanScalar>();
+ return selected.is_valid && selected.value ? batch : batch->Slice(0,
0);
+ }
+ ARROW_ASSIGN_OR_RAISE(
+ Datum filtered,
+ compute::Filter(batch, mask, compute::FilterOptions::Defaults(),
+ &exec_context));
+ return filtered.record_batch();
+ });
+}
+
+Future<std::optional<int64_t>> FilteredFragment::CountRows(
+ compute::Expression predicate, const std::shared_ptr<ScanOptions>&
options) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto combined,
+ compute::and_(std::move(predicate),
filter_).Bind(*options->dataset_schema));
+ return fragment_->CountRows(std::move(combined), options);
+}
+
+FilteredDataset::FilteredDataset(std::shared_ptr<Dataset> child,
+ compute::Expression filter)
+ : Dataset(child->schema(), child->partition_expression()),
+ child_(std::move(child)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<FilteredDataset>> FilteredDataset::Make(
+ std::shared_ptr<Dataset> child, compute::Expression filter) {
+ // Verify the filter produces a bool
+ ARROW_ASSIGN_OR_RAISE(auto bound, filter.Bind(*child->schema()));
+ if (!bound.type()->Equals(*boolean())) {
+ return Status::TypeError("FilteredDataset filter must be boolean, got ",
+ *bound.type());
+ }
+ // Store the unbound filter so it can be bound on-demand on the unified
schema of a
+ // UnionDataset.
+ return std::shared_ptr<FilteredDataset>(
+ new FilteredDataset(std::move(child), std::move(filter)));
+}
+
+Result<std::shared_ptr<Dataset>> FilteredDataset::ReplaceSchema(
+ std::shared_ptr<Schema> schema) const {
+ ARROW_ASSIGN_OR_RAISE(auto child, child_->ReplaceSchema(std::move(schema)));
+ ARROW_ASSIGN_OR_RAISE(auto filtered, Make(std::move(child), filter_));
+ return std::static_pointer_cast<Dataset>(std::move(filtered));
+}
+
+Result<FragmentIterator> FilteredDataset::GetFragmentsImpl(
+ compute::Expression predicate) {
+ // Compute a combined predicate that can be used for predicate-pushdown to
select
+ // fragments from the child Dataset.
+ ARROW_ASSIGN_OR_RAISE(auto combined_predicate,
+ compute::and_(std::move(predicate),
filter_).Bind(*schema_));
+ ARROW_ASSIGN_OR_RAISE(auto fragments,
+ child_->GetFragments(std::move(combined_predicate)));
+
+ // Convert the child's fragments to FilteredFragments that apply the
additional filter.
+ auto filter = filter_;
+ return MakeMapIterator(
+ [filter](std::shared_ptr<Fragment> fragment) ->
std::shared_ptr<Fragment> {
+ return std::make_shared<FilteredFragment>(std::move(fragment), filter);
+ },
+ std::move(fragments));
Review Comment:
Simplify `filter_` against each fragment's `partition_expression()` here so
standalone `frag.to_table()` / `frag.count_rows()` calls (without an explicit
`schema`) don't fail on partition columns absent from `physical_schema`.
##########
cpp/src/arrow/dataset/dataset.cc:
##########
@@ -321,6 +322,129 @@ Result<FragmentIterator>
UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}
+FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
+ compute::Expression filter)
+ : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
+ fragment_(std::move(fragment)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
+ return fragment_->ReadPhysicalSchema();
+}
+
+Status FilteredFragment::ClearCachedMetadata() {
+ RETURN_NOT_OK(Fragment::ClearCachedMetadata());
+ return fragment_->ClearCachedMetadata();
+}
+
+std::string FilteredFragment::ToString() const {
+ // The scanner uses ToString() for the __filename augmented field, so this
must
+ // match the wrapped fragment.
+ return fragment_->ToString();
+}
+
+Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) {
+ const auto& dataset_schema = options->dataset_schema;
+ ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(filter,
+ SimplifyWithGuarantee(std::move(filter),
partition_expression_));
+ if (!filter.IsSatisfiable()) {
+ return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
+ }
+
+ // Combine any existing filter in the scan options with the fragment's
filter,
+ // so the wrapped fragment can skip data that cannot match.
+ auto fragment_options = std::make_shared<ScanOptions>(*options);
+ ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
+ compute::and_(options->filter,
filter).Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(auto batches,
fragment_->ScanBatchesAsync(fragment_options));
+
+ // The filter might simplify to true if it only references partition fields,
+ // in which case we can skip filtering batches.
+ if (filter == compute::literal(true)) {
+ return batches;
+ }
Review Comment:
- Guard `options->dataset_schema == nullptr` (fall back to
`ReadPhysicalSchema()` or return `Status::Invalid`) for direct callers.
- Verify `filter` is boolean, since `FilteredFragment` can be constructed
directly and a `null`-typed field ref would pass `and_kleene` via implicit cast.
- Return `fragment_->ScanBatchesAsync(options)` early when `filter ==
literal(true)` to avoid copying `ScanOptions` and rebinding `options->filter`.
##########
cpp/src/arrow/dataset/dataset.cc:
##########
@@ -321,6 +322,129 @@ Result<FragmentIterator>
UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}
+FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
+ compute::Expression filter)
+ : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
+ fragment_(std::move(fragment)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
+ return fragment_->ReadPhysicalSchema();
+}
+
+Status FilteredFragment::ClearCachedMetadata() {
+ RETURN_NOT_OK(Fragment::ClearCachedMetadata());
+ return fragment_->ClearCachedMetadata();
+}
+
+std::string FilteredFragment::ToString() const {
+ // The scanner uses ToString() for the __filename augmented field, so this
must
+ // match the wrapped fragment.
+ return fragment_->ToString();
+}
+
+Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) {
+ const auto& dataset_schema = options->dataset_schema;
+ ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(filter,
+ SimplifyWithGuarantee(std::move(filter),
partition_expression_));
+ if (!filter.IsSatisfiable()) {
+ return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
+ }
+
+ // Combine any existing filter in the scan options with the fragment's
filter,
+ // so the wrapped fragment can skip data that cannot match.
+ auto fragment_options = std::make_shared<ScanOptions>(*options);
+ ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
+ compute::and_(options->filter,
filter).Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(auto batches,
fragment_->ScanBatchesAsync(fragment_options));
+
+ // The filter might simplify to true if it only references partition fields,
+ // in which case we can skip filtering batches.
+ if (filter == compute::literal(true)) {
+ return batches;
+ }
+
+ // Apply the fragment's filter to scanned rows.
+ // The filter in the scan options is only used for data pruning at this point
+ // and will be used to filter rows later, so don't duplicate that work.
+ auto guarantee = partition_expression_;
+ auto pool = options->pool;
+ return MakeMappedGenerator(
+ std::move(batches),
+ [filter, dataset_schema, guarantee, pool](const
std::shared_ptr<RecordBatch>& batch)
+ -> Result<std::shared_ptr<RecordBatch>> {
+ compute::ExecContext exec_context(pool);
+ ARROW_ASSIGN_OR_RAISE(auto exec_batch,
+ compute::MakeExecBatch(*dataset_schema, batch,
guarantee));
+ ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression(
+ filter, exec_batch,
&exec_context));
+ if (mask.is_scalar()) {
+ const auto& selected = mask.scalar_as<BooleanScalar>();
+ return selected.is_valid && selected.value ? batch : batch->Slice(0,
0);
+ }
+ ARROW_ASSIGN_OR_RAISE(
+ Datum filtered,
+ compute::Filter(batch, mask, compute::FilterOptions::Defaults(),
+ &exec_context));
+ return filtered.record_batch();
+ });
+}
+
+Future<std::optional<int64_t>> FilteredFragment::CountRows(
+ compute::Expression predicate, const std::shared_ptr<ScanOptions>&
options) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto combined,
+ compute::and_(std::move(predicate),
filter_).Bind(*options->dataset_schema));
+ return fragment_->CountRows(std::move(combined), options);
+}
+
+FilteredDataset::FilteredDataset(std::shared_ptr<Dataset> child,
+ compute::Expression filter)
+ : Dataset(child->schema(), child->partition_expression()),
+ child_(std::move(child)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<FilteredDataset>> FilteredDataset::Make(
+ std::shared_ptr<Dataset> child, compute::Expression filter) {
+ // Verify the filter produces a bool
+ ARROW_ASSIGN_OR_RAISE(auto bound, filter.Bind(*child->schema()));
+ if (!bound.type()->Equals(*boolean())) {
Review Comment:
Validate `child != nullptr` and `filter.is_valid()` before binding so
invalid inputs return `Status::Invalid` instead of crashing.
##########
cpp/src/arrow/dataset/dataset.cc:
##########
@@ -321,6 +322,129 @@ Result<FragmentIterator>
UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}
+FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
+ compute::Expression filter)
+ : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
+ fragment_(std::move(fragment)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
+ return fragment_->ReadPhysicalSchema();
+}
+
+Status FilteredFragment::ClearCachedMetadata() {
+ RETURN_NOT_OK(Fragment::ClearCachedMetadata());
+ return fragment_->ClearCachedMetadata();
+}
+
+std::string FilteredFragment::ToString() const {
+ // The scanner uses ToString() for the __filename augmented field, so this
must
+ // match the wrapped fragment.
+ return fragment_->ToString();
+}
+
+Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) {
+ const auto& dataset_schema = options->dataset_schema;
+ ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(filter,
+ SimplifyWithGuarantee(std::move(filter),
partition_expression_));
+ if (!filter.IsSatisfiable()) {
+ return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
+ }
+
+ // Combine any existing filter in the scan options with the fragment's
filter,
+ // so the wrapped fragment can skip data that cannot match.
+ auto fragment_options = std::make_shared<ScanOptions>(*options);
+ ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
+ compute::and_(options->filter,
filter).Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(auto batches,
fragment_->ScanBatchesAsync(fragment_options));
+
+ // The filter might simplify to true if it only references partition fields,
+ // in which case we can skip filtering batches.
+ if (filter == compute::literal(true)) {
+ return batches;
+ }
+
+ // Apply the fragment's filter to scanned rows.
+ // The filter in the scan options is only used for data pruning at this point
+ // and will be used to filter rows later, so don't duplicate that work.
+ auto guarantee = partition_expression_;
+ auto pool = options->pool;
+ return MakeMappedGenerator(
+ std::move(batches),
+ [filter, dataset_schema, guarantee, pool](const
std::shared_ptr<RecordBatch>& batch)
+ -> Result<std::shared_ptr<RecordBatch>> {
+ compute::ExecContext exec_context(pool);
+ ARROW_ASSIGN_OR_RAISE(auto exec_batch,
+ compute::MakeExecBatch(*dataset_schema, batch,
guarantee));
+ ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression(
+ filter, exec_batch,
&exec_context));
+ if (mask.is_scalar()) {
+ const auto& selected = mask.scalar_as<BooleanScalar>();
+ return selected.is_valid && selected.value ? batch : batch->Slice(0,
0);
+ }
+ ARROW_ASSIGN_OR_RAISE(
+ Datum filtered,
+ compute::Filter(batch, mask, compute::FilterOptions::Defaults(),
+ &exec_context));
+ return filtered.record_batch();
+ });
+}
+
+Future<std::optional<int64_t>> FilteredFragment::CountRows(
+ compute::Expression predicate, const std::shared_ptr<ScanOptions>&
options) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto combined,
+ compute::and_(std::move(predicate),
filter_).Bind(*options->dataset_schema));
+ return fragment_->CountRows(std::move(combined), options);
+}
Review Comment:
Simplify `combined` against `partition_expression_` and return `0` when
unsatisfiable, matching `FileFragment::CountRows` — otherwise
`InMemoryFragment::CountRows` overcounts unsatisfiable constant filters (e.g.
`literal(false)`) and misses the fast path for partition-only filters.
Also handle `options->dataset_schema == nullptr` (direct callers like
`TestCountRows` pass default `ScanOptions`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]