aokolnychyi commented on a change in pull request #35395: URL: https://github.com/apache/spark/pull/35395#discussion_r822927633
########## File path: sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala ########## @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog + +import java.util + +import org.apache.spark.sql.connector.distributions.{Distribution, Distributions} +import org.apache.spark.sql.connector.expressions.{FieldReference, LogicalExpressions, NamedReference, SortDirection, SortOrder, Transform} +import org.apache.spark.sql.connector.read.{Scan, ScanBuilder} +import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, RequiresDistributionAndOrdering, RowLevelOperation, RowLevelOperationBuilder, RowLevelOperationInfo, Write, WriteBuilder, WriterCommitMessage} +import org.apache.spark.sql.connector.write.RowLevelOperation.Command +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +class InMemoryRowLevelOperationTable( + name: String, + schema: StructType, + partitioning: Array[Transform], + properties: util.Map[String, String]) + extends InMemoryTable(name, schema, partitioning, properties) with SupportsRowLevelOperations { + + override def newRowLevelOperationBuilder( + info: RowLevelOperationInfo): RowLevelOperationBuilder = { + () => PartitionBasedOperation(info.command) + } + + case class PartitionBasedOperation(command: Command) extends RowLevelOperation { + private final val PARTITION_COLUMN_REF = FieldReference(PartitionKeyColumn.name) + + var configuredScan: InMemoryBatchScan = _ + + override def requiredMetadataAttributes(): Array[NamedReference] = { + Array(PARTITION_COLUMN_REF) + } + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { + new InMemoryScanBuilder(schema) { + override def build: Scan = { + val scan = super.build() + configuredScan = scan.asInstanceOf[InMemoryBatchScan] + scan + } + } + } + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = new WriteBuilder { + + override def build(): Write = new Write with RequiresDistributionAndOrdering { + override def requiredDistribution(): Distribution = { + Distributions.clustered(Array(PARTITION_COLUMN_REF)) + } + + override def requiredOrdering(): Array[SortOrder] = { + Array[SortOrder]( + LogicalExpressions.sort( + PARTITION_COLUMN_REF, + SortDirection.ASCENDING, + SortDirection.ASCENDING.defaultNullOrdering()) + ) + } + + override def toBatch: BatchWrite = PartitionBasedReplaceData(configuredScan) + + override def description(): String = "InMemoryWrite" + } + } + + override def description(): String = "InMemoryPartitionReplaceOperation" + } + + private case class PartitionBasedReplaceData(scan: InMemoryBatchScan) extends TestBatchWrite { + + override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized { + val newData = messages.map(_.asInstanceOf[BufferedRows]) + val readRows = scan.data.flatMap(_.asInstanceOf[BufferedRows].rows) Review comment: This is the place where the writer can obtain unique groups IDs that were read (i.e. files in case of Delta). Those group IDs can be filtered in two ways: - static - runtime The proposal is that Spark will first should push down whatever possible conditions so that data sources can use static filters to prune partitions and files using metadata. You are right subqueries won't be pushed down but there may be other simple conditions. That gives us a set of files that **_may_** potentially match (still has false positives). In the example you provided, static filtering is not possible since we only have a subquery. Next, Spark should scan the potentially matching files using a separate filtering query to reduce the amount of data to rewrite. That should give us the minimum set of data files to rewrite. I agree with the filtering subquery you provided. I think we are talking about the same algorithm but I still believe the query you mentioned can be executed dynamically and the unique files can be collected and passed as a runtime IN filter. Let me show how plans can look like for a DELETE statement with an IN subquery. ``` DELETE FROM t WHERE id IN (SELECT * FROM deleted_id) ``` ``` == Optimized Logical Plan == ReplaceData RelationV2[id#66, dep#67] t +- Project [id#66, dep#67] +- Filter NOT (exists#86 <=> true) +- Join ExistenceJoin(exists#86), (id#66 = value#19) :- Project [id#66, dep#67] : +- Filter dynamicpruning#85 [_file_name#70] : : +- Project [_file_name#83] : : +- Join LeftSemi, (id#81 = value#72) : : :- Project [id#81, _file_name#83] : : : +- RelationV2[id#81, dep#82, _file_name#83] t : : +- LocalRelation [value#72] : +- RelationV2[id#66, dep#67, _file_name#70] t +- LocalRelation [value#19] ``` Just like you said, the filtering subquery would be planned as a scalable left semi join. Does that make sense? I think we are talking about the same algorithm, just different representation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
