zanmato1984 commented on code in PR #55665:
URL: https://github.com/apache/arrow/pull/55665#discussion_r4648866867
##########
python/pyarrow/dataset.py:
##########
@@ -500,16 +501,9 @@ def _union_dataset(children, schema=None, **kwargs):
# unify the children datasets' schemas
schema = pa.unify_schemas([child.schema for child in children])
- for child in children:
- if getattr(child, "_scan_options", None):
- raise ValueError(
- "Creating an UnionDataset from filtered or projected Datasets "
- "is currently not supported. Union the unfiltered datasets "
- "and apply the filter to the resulting union."
- )
-
- # create datasets with the requested schema
- children = [child.replace_schema(schema) for child in children]
+ # create datasets with the requested schema, keeping each child's own
filter
+ children = [child._with_filter_attached().replace_schema(schema)
Review Comment:
For a child with `x = [0, 1]` filtered by `pc.field("x") == 1`,
`ds.dataset([child])` returns `[1]`, but `ds.UnionDataset(child.schema,
[child])` returns `[0, 1]`. The direct constructor already ignored the filter
before this change; attaching it only in the factory leaves the two public
entry points with different results.
Could both paths share the filter-attachment logic and test the same
filtered child? If direct construction is intentionally unsupported for
filtered children, it should raise rather than return rows the child excludes.
##########
cpp/src/arrow/dataset/dataset.cc:
##########
@@ -321,6 +322,129 @@ Result<FragmentIterator>
UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}
+FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
+ compute::Expression filter)
+ : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
+ fragment_(std::move(fragment)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
+ return fragment_->ReadPhysicalSchema();
+}
+
+Status FilteredFragment::ClearCachedMetadata() {
+ RETURN_NOT_OK(Fragment::ClearCachedMetadata());
+ return fragment_->ClearCachedMetadata();
+}
+
+std::string FilteredFragment::ToString() const {
+ // The scanner uses ToString() for the __filename augmented field, so this
must
+ // match the wrapped fragment.
+ return fragment_->ToString();
+}
+
+Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) {
+ const auto& dataset_schema = options->dataset_schema;
+ ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(filter,
+ SimplifyWithGuarantee(std::move(filter),
partition_expression_));
+ if (!filter.IsSatisfiable()) {
+ return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
+ }
+
+ // Combine any existing filter in the scan options with the fragment's
filter,
+ // so the wrapped fragment can skip data that cannot match.
+ auto fragment_options = std::make_shared<ScanOptions>(*options);
+ ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
+ compute::and_(options->filter,
filter).Bind(*dataset_schema));
+ ARROW_ASSIGN_OR_RAISE(auto batches,
fragment_->ScanBatchesAsync(fragment_options));
+
+ // The filter might simplify to true if it only references partition fields,
+ // in which case we can skip filtering batches.
+ if (filter == compute::literal(true)) {
+ return batches;
+ }
+
+ // Apply the fragment's filter to scanned rows.
+ // The filter in the scan options is only used for data pruning at this point
+ // and will be used to filter rows later, so don't duplicate that work.
+ auto guarantee = partition_expression_;
+ auto pool = options->pool;
+ return MakeMappedGenerator(
+ std::move(batches),
+ [filter, dataset_schema, guarantee, pool](const
std::shared_ptr<RecordBatch>& batch)
+ -> Result<std::shared_ptr<RecordBatch>> {
+ compute::ExecContext exec_context(pool);
+ ARROW_ASSIGN_OR_RAISE(auto exec_batch,
+ compute::MakeExecBatch(*dataset_schema, batch,
guarantee));
+ ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression(
+ filter, exec_batch,
&exec_context));
+ if (mask.is_scalar()) {
+ const auto& selected = mask.scalar_as<BooleanScalar>();
+ return selected.is_valid && selected.value ? batch : batch->Slice(0,
0);
+ }
+ ARROW_ASSIGN_OR_RAISE(
+ Datum filtered,
+ compute::Filter(batch, mask, compute::FilterOptions::Defaults(),
+ &exec_context));
+ return filtered.record_batch();
+ });
+}
+
+Future<std::optional<int64_t>> FilteredFragment::CountRows(
+ compute::Expression predicate, const std::shared_ptr<ScanOptions>&
options) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto combined,
+ compute::and_(std::move(predicate),
filter_).Bind(*options->dataset_schema));
+ return fragment_->CountRows(std::move(combined), options);
+}
+
+FilteredDataset::FilteredDataset(std::shared_ptr<Dataset> child,
+ compute::Expression filter)
+ : Dataset(child->schema(), child->partition_expression()),
+ child_(std::move(child)),
+ filter_(std::move(filter)) {}
+
+Result<std::shared_ptr<FilteredDataset>> FilteredDataset::Make(
+ std::shared_ptr<Dataset> child, compute::Expression filter) {
+ // Verify the filter produces a bool
+ ARROW_ASSIGN_OR_RAISE(auto bound, filter.Bind(*child->schema()));
+ if (!bound.type()->Equals(*boolean())) {
+ return Status::TypeError("FilteredDataset filter must be boolean, got ",
+ *bound.type());
+ }
+ // Store the unbound filter so it can be bound on-demand on the unified
schema of a
+ // UnionDataset.
+ return std::shared_ptr<FilteredDataset>(
+ new FilteredDataset(std::move(child), std::move(filter)));
+}
+
+Result<std::shared_ptr<Dataset>> FilteredDataset::ReplaceSchema(
+ std::shared_ptr<Schema> schema) const {
+ ARROW_ASSIGN_OR_RAISE(auto child, child_->ReplaceSchema(std::move(schema)));
+ ARROW_ASSIGN_OR_RAISE(auto filtered, Make(std::move(child), filter_));
Review Comment:
`field(0)` refers to the first column of the original child, not the first
column of the union. For a child with columns `[x, y]` and rows `(x=0, y=1)`
and `(x=1, y=0)`, `field(0) == 1` selects the second row. When the union uses
`[y, x]`, rebinding the same expression here makes it test `y == 1` and return
the first row instead.
Could we resolve the positional reference against the original schema and
preserve the referenced field when replacing the schema? A regression test
should check the returned `x` and `y` values: a row-count assertion would pass
in both cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]