viirya commented on a change in pull request #35395: URL: https://github.com/apache/spark/pull/35395#discussion_r802214039
########## File path: sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRowLevelOperations.java ########## @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import org.apache.spark.annotation.Experimental; +import org.apache.spark.sql.connector.write.RowLevelOperationBuilder; +import org.apache.spark.sql.connector.write.RowLevelOperation; +import org.apache.spark.sql.connector.write.RowLevelOperationInfo; + +/** + * A mix-in interface for {@link Table} row-level operations support. Data sources can implement + * this interface to indicate they support rewriting data for DELETE, UPDATE, MERGE operations. + * + * @since 3.3.0 + */ +@Experimental +public interface SupportsRowLevelOperations extends Table { + /** + * Returns a {@link RowLevelOperationBuilder} to build a {@link RowLevelOperation}. + * Spark will call this method while planning DELETE, UPDATE and MERGE operations + * that require rewriting data. + * + * @param info the row-level operation info such command (e.g. DELETE) and options Review comment: typo? "for" such command? ########## File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelCommand.scala ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import scala.collection.mutable + +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, ExprId, V2ExpressionUtils} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations +import org.apache.spark.sql.connector.write.{RowLevelOperation, RowLevelOperationInfoImpl, RowLevelOperationTable} +import org.apache.spark.sql.connector.write.RowLevelOperation.Command +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +trait RewriteRowLevelCommand extends Rule[LogicalPlan] { + + protected def buildRowLevelOperation( + table: SupportsRowLevelOperations, + command: Command): RowLevelOperation = { + val info = RowLevelOperationInfoImpl(command, CaseInsensitiveStringMap.empty()) + val builder = table.newRowLevelOperationBuilder(info) + builder.build() + } Review comment: Hmm, don't we need to pass any option map into the `RowLevelOperationInfo`? ########## File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala ########## @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.expressions.{EqualNullSafe, Expression, Literal, Not} +import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, Filter, LogicalPlan, ReplaceData} +import org.apache.spark.sql.connector.catalog.{SupportsDelete, SupportsRowLevelOperations} +import org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE +import org.apache.spark.sql.connector.write.RowLevelOperationTable +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + +/** + * Assigns a rewrite plan for v2 tables that support rewriting data to handle DELETE statements. + * + * If a table implements [[SupportsDelete]] and [[SupportsRowLevelOperations]], this rule assigns + * a rewrite plan but the optimizer will check whether this particular DELETE statement can be + * handled by simply passing delete filters to the connector. If yes, the optimizer will then + * discard the rewrite plan. + */ +object RewriteDeleteFromTable extends RewriteRowLevelCommand { + + override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { + case d @ DeleteFromTable(aliasedTable, cond, None) if d.resolved => + EliminateSubqueryAliases(aliasedTable) match { + case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) => Review comment: Don't need to check `SupportsDelete`? ########## File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala ########## @@ -176,6 +176,52 @@ object OverwritePartitionsDynamic { } } +/** + * Replace data in an existing table. + */ +case class ReplaceData( + table: NamedRelation, + query: LogicalPlan, + originalTable: NamedRelation, + write: Option[Write] = None) extends V2WriteCommand with NestedCommand { Review comment: Can you add some param docs for these parameters? ########## File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala ########## @@ -55,3 +55,9 @@ trait AnalysisOnlyCommand extends Command { // on the `AnalysisContext` def markAsAnalyzed(analysisContext: AnalysisContext): LogicalPlan } + +/** + * A command that is nested within another command after the analysis and does not have to be + * executed eagerly. Such commands will be either removed or made top-level in the optimizer. + */ +trait NestedCommand extends Command Review comment: So we should not see such `NestedCommand` after optimization, right? ########## File path: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala ########## @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.expressions.{EqualNullSafe, Expression, Literal, Not} +import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, Filter, LogicalPlan, ReplaceData} +import org.apache.spark.sql.connector.catalog.{SupportsDelete, SupportsRowLevelOperations} +import org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE +import org.apache.spark.sql.connector.write.RowLevelOperationTable +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + +/** + * Assigns a rewrite plan for v2 tables that support rewriting data to handle DELETE statements. + * + * If a table implements [[SupportsDelete]] and [[SupportsRowLevelOperations]], this rule assigns + * a rewrite plan but the optimizer will check whether this particular DELETE statement can be + * handled by simply passing delete filters to the connector. If yes, the optimizer will then + * discard the rewrite plan. Review comment: Where the optimizer will do the check and discard the rewrite plan? In `ReplaceRewrittenRowLevelCommands`, seems it simply gets the rewrite plan of the `RowLevelCommand` (e.g. `DeleteFromTable`). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
