JingsongLi commented on code in PR #11265:
URL: https://github.com/apache/paimon/pull/11265#discussion_r4624907543
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsRuntimeFiltering.scala:
##########
@@ -69,12 +163,60 @@ trait PaimonSupportsRuntimeFiltering extends BaseScan with
SupportsRuntimeV2Filt
val runtimePartitionFilters = predicates.toSeq
.flatMap(converter.convert(_))
.map(PartitionPredicate.fromPredicate(partitionType, _))
+ val scalarFilters =
+ if (runtimeIndexFields.isEmpty) Seq.empty
+ else {
+ val dataConverter = SparkV2FilterConverter(table.rowType())
+ predicates
+ .filter {
+ predicate =>
+ val children = predicate.children()
+ // Bound index work after Spark has collected the keys; the join
remains intact.
+ predicate.name() == "IN" && children.nonEmpty && children.length
<= 10001 &&
+ (children.head match {
+ case ref: NamedReference =>
+ ref.fieldNames().length == 1 && runtimeIndexFields.contains(
+ ref.fieldNames().head) &&
+ children.tail.forall {
+ case literal: Literal[_] =>
+ literal.dataType() == SparkTypeUtils.fromPaimonType(
+
table.rowType().getField(ref.fieldNames().head).`type`())
+ case _ => false
+ }
+ case _ => false
+ }) && children.tail.forall(_.isInstanceOf[Literal[_]]) &&
+ children.tail.iterator.map {
+ case literal: Literal[_] =>
+ Option(literal.value())
+
.map(_.toString.getBytes(StandardCharsets.UTF_8).length.toLong)
+ .getOrElse(0L)
+ }.sum <= 1024 * 1024
+ }
+ .flatMap {
+ predicate =>
+ if (predicate.children().length == 1) {
+ val name =
predicate.children().head.asInstanceOf[NamedReference].fieldNames().head
+ Some(
+ new PredicateBuilder(table.rowType()).in(
+ table.rowType().getFieldIndex(name),
+ java.util.Collections.emptyList[Object]()))
+ } else {
+ dataConverter.convert(predicate)
+ }
+ }
+ .toSeq
+ }
if (runtimePartitionFilters.nonEmpty) {
pushedRuntimePartitionFilters.appendAll(runtimePartitionFilters)
readBuilder.withPartitionFilter(
PartitionPredicate.and(
(pushedPartitionFilters ++
pushedRuntimePartitionFilters).toList.asJava))
- // set inputPartitions null to trigger to get the new splits.
+ }
+ if (scalarFilters.nonEmpty) {
+ // Keep the same builder: Spark may retain the original batch's reader
factory.
+ readBuilder.withFilter(scalarFilters.asJava)
Review Comment:
**[P1] Keep matching rows outside the composite index actually chosen**
A complete single-column index qualifies `k` for runtime filtering, but
combining runtime `IN(k)` with a static predicate lets the planner choose a
different, incomplete composite index. With the default FAST mode I built
`btree(k,c)` on fact IDs 0–99, appended `(100,2,10)`, and then built a complete
`btree(k)`. A broadcast join on `k` for dimension key 2 with static `f.c = 10`
returned 11 fact rows with dynamic pruning off and 10 with it on, silently
omitting ID 100. I verified both planning-time indexing and
`global-index.query-in-reader.enabled=true`.
Either constrain the runtime path to the validated index or enforce full
coverage/fallback for the index plan actually selected.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsRuntimeFiltering.scala:
##########
@@ -38,6 +47,93 @@ trait PaimonSupportsRuntimeFiltering extends BaseScan with
SupportsRuntimeV2Filt
private var _inputSplits: Array[Split] = _
private var _inputPartitions: Seq[PaimonInputPartition] = _
+ // Pin metadata discovery and every subsequent scan to the same snapshot. In
particular,
+ // replanning after the build side completes must not pick up a concurrent
append.
+ private lazy val runtimeReadSnapshot: Option[(FileStoreTable, Snapshot)] = {
+ val eligibleScan = this match {
+ case scan: PaimonScan => !scan.preserveDataGrouping
+ case _ => false
+ }
+ if (
+ !eligibleScan || pushedLimit.isDefined || pushedTopN.isDefined ||
+ pushedVectorSearch.isDefined || pushedHybridSearch.isDefined ||
+ pushedFullTextSearch.isDefined || !coreOptions.dataEvolutionEnabled() ||
+ !coreOptions.globalIndexEnabled() || coreOptions.queryAuthEnabled() ||
+ coreOptions.ignoreIndexColumnUpdate() ||
+ coreOptions.startupMode() != StartupMode.LATEST_FULL
+ ) {
+ None
+ } else {
+ table match {
+ case fileTable: FileStoreTable if fileTable.primaryKeys().isEmpty =>
+ Option(fileTable.snapshotManager().latestSnapshot()).map {
+ snapshot =>
+ val pinned = fileTable.copyWithoutTimeTravel(
+ Map(
+ CoreOptions.SCAN_MODE.key() -> "from-snapshot",
+ CoreOptions.SCAN_SNAPSHOT_ID.key() ->
snapshot.id().toString).asJava)
+ pinned -> snapshot
+ }
+ case _ => None
+ }
+ }
+ }
+
+ // Ordinary scans do not pay for coverage and freshness discovery. Spark
requests these
+ // attributes only when considering runtime filtering for a join.
+ private lazy val runtimeIndexFields: Set[String] =
runtimeReadSnapshot.toSeq.flatMap {
+ case (fileTable, snapshot) =>
+ val indexEntries = Seq("btree", "bitmap")
+ .flatMap {
+ indexType => fileTable.store().newIndexFileHandler().scan(snapshot,
indexType).asScala
+ }
+ .filter(
+ e =>
+ e.indexFile().globalIndexMeta() != null &&
+ e.indexFile().globalIndexMeta().getIndexedFieldIds.size() == 1)
+ val indexFiles = indexEntries.map(_.indexFile())
+ val indexedIds = indexFiles.map(_.globalIndexMeta().indexFieldId()).toSet
+ val coverage = new DataEvolutionGlobalIndexCoverage(
+ fileTable,
+ snapshot,
+ null,
+ indexFiles.asJava,
+ GlobalIndexSearchMode.FULL)
+ readTableRowType.getFields.asScala
+ .filter {
+ field =>
+ val supportedType = field.`type`().getTypeRoot match {
+ case DataTypeRoot.BOOLEAN | DataTypeRoot.TINYINT |
DataTypeRoot.SMALLINT |
+ DataTypeRoot.INTEGER | DataTypeRoot.BIGINT |
DataTypeRoot.CHAR |
+ DataTypeRoot.VARCHAR | DataTypeRoot.DATE =>
+ true
+ case _ => false
+ }
+ val fieldEntries =
+
indexEntries.filter(_.indexFile().globalIndexMeta().indexFieldId() ==
field.id())
+ supportedType && indexedIds.contains(field.id()) &&
+ !fileTable.partitionKeys().contains(field.name()) &&
+ coverage.unindexedRanges(field.id()).isEmpty &&
+ fieldEntries.forall(
+ e =>
+ DataEvolutionIndexSourceMeta.isDataEvolutionMeta(
+ e.indexFile().globalIndexMeta().sourceMeta())) &&
+ // A past writer may have used IGNORE only as a dynamic option.
+ DataEvolutionGlobalIndexRefreshPlanner
+ .findIndexesToRefresh(
+ fileTable,
+ snapshot,
Review Comment:
**[P2] Defer full freshness scans until Spark applies a scalar runtime
predicate**
Spark 3.5.9 calls `filterAttributes()` before deciding whether a join can
produce a runtime predicate. In a real Data Evolution table with three
partition manifests, BTree indexes on two projected fields, and a static filter
selecting just one partition, one capability call opened the index manifest
twice and each of the three data manifests twice. An ordinary inner join with
an unfiltered dimension produced `RuntimeFilters: []`, so discovery can do this
work even when no pruning is applied. At scale the extra reads fan out across
the entire fact table for each candidate field. Defer the freshness checks
until a usable scalar predicate arrives, share index manifest reads, and apply
the static partition scope where possible.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]