JingsongLi commented on code in PR #11265:
URL: https://github.com/apache/paimon/pull/11265#discussion_r4625055062


##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsRuntimeFiltering.scala:
##########
@@ -38,6 +47,93 @@ trait PaimonSupportsRuntimeFiltering extends BaseScan with 
SupportsRuntimeV2Filt
   private var _inputSplits: Array[Split] = _
   private var _inputPartitions: Seq[PaimonInputPartition] = _
 
+  // Pin metadata discovery and every subsequent scan to the same snapshot. In 
particular,
+  // replanning after the build side completes must not pick up a concurrent 
append.
+  private lazy val runtimeReadSnapshot: Option[(FileStoreTable, Snapshot)] = {
+    val eligibleScan = this match {
+      case scan: PaimonScan => !scan.preserveDataGrouping
+      case _ => false
+    }
+    if (
+      !eligibleScan || pushedLimit.isDefined || pushedTopN.isDefined ||
+      pushedVectorSearch.isDefined || pushedHybridSearch.isDefined ||
+      pushedFullTextSearch.isDefined || !coreOptions.dataEvolutionEnabled() ||
+      !coreOptions.globalIndexEnabled() || coreOptions.queryAuthEnabled() ||
+      coreOptions.ignoreIndexColumnUpdate() ||
+      coreOptions.startupMode() != StartupMode.LATEST_FULL
+    ) {
+      None
+    } else {
+      table match {
+        case fileTable: FileStoreTable if fileTable.primaryKeys().isEmpty =>
+          Option(fileTable.snapshotManager().latestSnapshot()).map {
+            snapshot =>
+              val pinned = fileTable.copyWithoutTimeTravel(
+                Map(
+                  CoreOptions.SCAN_MODE.key() -> "from-snapshot",
+                  CoreOptions.SCAN_SNAPSHOT_ID.key() -> 
snapshot.id().toString).asJava)
+              pinned -> snapshot
+          }
+        case _ => None
+      }
+    }
+  }
+
+  // Ordinary scans do not pay for coverage and freshness discovery. Spark 
requests these
+  // attributes only when considering runtime filtering for a join.
+  private lazy val runtimeIndexFields: Set[String] = 
runtimeReadSnapshot.toSeq.flatMap {
+    case (fileTable, snapshot) =>
+      val indexEntries = Seq("btree", "bitmap")
+        .flatMap {
+          indexType => fileTable.store().newIndexFileHandler().scan(snapshot, 
indexType).asScala
+        }
+        .filter(
+          e =>
+            e.indexFile().globalIndexMeta() != null &&
+              e.indexFile().globalIndexMeta().getIndexedFieldIds.size() == 1)
+      val indexFiles = indexEntries.map(_.indexFile())
+      val indexedIds = indexFiles.map(_.globalIndexMeta().indexFieldId()).toSet
+      val coverage = new DataEvolutionGlobalIndexCoverage(
+        fileTable,
+        snapshot,
+        null,
+        indexFiles.asJava,
+        GlobalIndexSearchMode.FULL)
+      readTableRowType.getFields.asScala
+        .filter {
+          field =>
+            val supportedType = field.`type`().getTypeRoot match {
+              case DataTypeRoot.BOOLEAN | DataTypeRoot.TINYINT | 
DataTypeRoot.SMALLINT |
+                  DataTypeRoot.INTEGER | DataTypeRoot.BIGINT | 
DataTypeRoot.CHAR |
+                  DataTypeRoot.VARCHAR | DataTypeRoot.DATE =>
+                true
+              case _ => false
+            }
+            val fieldEntries =
+              
indexEntries.filter(_.indexFile().globalIndexMeta().indexFieldId() == 
field.id())
+            supportedType && indexedIds.contains(field.id()) &&
+            !fileTable.partitionKeys().contains(field.name()) &&
+            coverage.unindexedRanges(field.id()).isEmpty &&
+            fieldEntries.forall(
+              e =>
+                DataEvolutionIndexSourceMeta.isDataEvolutionMeta(
+                  e.indexFile().globalIndexMeta().sourceMeta())) &&
+            // A past writer may have used IGNORE only as a dynamic option.
+            DataEvolutionGlobalIndexRefreshPlanner
+              .findIndexesToRefresh(
+                fileTable,
+                snapshot,
+                null,
+                fieldEntries.asJava,
+                java.util.Collections.singletonList(field))
+              .isEmpty
+        }
+        .map(_.name())
+        .toSet
+  }.toSet
+
+  override protected def readTable: Table = 
runtimeReadSnapshot.map(_._1).getOrElse(table)

Review Comment:
   **[P1] Keep the runtime-filter trait binary compatible on Spark 3.2 and 3.3**
   
   Spark 3.2 and 3.3 each ship a connector-local trait with the same fully 
qualified name. The common `PaimonBaseScan` now forwards to the new trait 
method `readTable$`, but both local variants lack it. 
`SparkGenericCatalogTest.testPaimonTable` fails before any runtime filter is 
applied in [Scala 2.12 
CI](https://github.com/apache/paimon/actions/runs/44454667392/job/135326784703) 
and [Scala 2.13 
CI](https://github.com/apache/paimon/actions/runs/44454667392/job/135326784735).
 Keep the bridge in each connector variant or keep the method dispatch in one 
common class, and exercise the packaged connectors in the regression.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to