This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9fb989921 [GLUTEN-6834][CORE] Remove unused DDL plan that doesn't
correspond to Substrait spec (#6833)
9fb989921 is described below
commit 9fb989921586b992da4db55669ebf230f5cf6330
Author: David Sisson <[email protected]>
AuthorDate: Wed Aug 14 21:29:22 2024 -0700
[GLUTEN-6834][CORE] Remove unused DDL plan that doesn't correspond to
Substrait spec (#6833)
---
.../apache/gluten/substrait/ddlplan/DllNode.java | 39 ---------------
.../gluten/substrait/ddlplan/DllPlanNode.java | 29 -----------
.../substrait/ddlplan/DllTransformContext.scala | 28 -----------
.../substrait/ddlplan/InsertOutputBuilder.java | 26 ----------
.../gluten/substrait/ddlplan/InsertOutputNode.java | 56 ----------------------
.../gluten/substrait/ddlplan/InsertPlanNode.java | 50 -------------------
.../resources/substrait/proto/substrait/ddl.proto | 25 ----------
.../apache/gluten/substrait/SubstraitContext.scala | 9 ----
8 files changed, 262 deletions(-)
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java
deleted file mode 100644
index e5878e0e3..000000000
--- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.substrait.ddlplan;
-
-import io.substrait.proto.Dll;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class DllNode implements Serializable {
-
- private final List<DllPlanNode> dllPlans;
-
- public DllNode(List<DllPlanNode> dllPlans) {
- this.dllPlans = dllPlans;
- }
-
- public Dll toProtobuf() {
- Dll.Builder dllBuilder = Dll.newBuilder();
- for (DllPlanNode dllPlanNode : dllPlans) {
- dllBuilder.addDllPlan(dllPlanNode.toProtobuf());
- }
- return dllBuilder.build();
- }
-}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java
deleted file mode 100644
index 80c57eaa8..000000000
---
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.substrait.ddlplan;
-
-import io.substrait.proto.DllPlan;
-
-/** Contains helper functions for constructing substrait relations. */
-public interface DllPlanNode {
- /**
- * Converts a Expression into a protobuf.
- *
- * @return A rel protobuf
- */
- DllPlan toProtobuf();
-}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala
deleted file mode 100644
index d63bb839f..000000000
---
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gluten.substrait.ddlplan
-
-import org.apache.gluten.substrait.SubstraitContext
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DllTransformContext(
- inputAttributes: Seq[Attribute],
- outputAttributes: Seq[Attribute],
- root: DllNode,
- substraitContext: SubstraitContext = null)
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java
deleted file mode 100644
index 75146de0c..000000000
---
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.substrait.ddlplan;
-
-public class InsertOutputBuilder {
- private InsertOutputBuilder() {}
-
- public static InsertOutputNode makeInsertOutputNode(
- Long partsNum, String database, String tableName, String relativePath) {
- return new InsertOutputNode(partsNum, database, tableName, relativePath);
- }
-}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java
deleted file mode 100644
index c5804e4e2..000000000
---
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.substrait.ddlplan;
-
-import com.google.protobuf.Any;
-import com.google.protobuf.ByteString;
-import io.substrait.proto.ReadRel;
-
-import java.io.Serializable;
-
-public class InsertOutputNode implements Serializable {
- private static final String MERGE_TREE = "MergeTree;";
- private Long partsNum;
- private String database = null;
- private String tableName = null;
- private String relativePath = null;
- private StringBuffer extensionTableStr = new StringBuffer(MERGE_TREE);
-
- InsertOutputNode(Long partsNum, String database, String tableName, String
relativePath) {
- this.partsNum = partsNum;
- this.database = database;
- this.tableName = tableName;
- this.relativePath = relativePath;
- // MergeTree;{database}\n{table}\n{relative_path}\n{min_part}\n{max_part}\n
- extensionTableStr
- .append(database)
- .append("\n")
- .append(tableName)
- .append("\n")
- .append(relativePath)
- .append("\n")
- .append(this.partsNum)
- .append("\n");
- }
-
- public ReadRel.ExtensionTable toProtobuf() {
- ReadRel.ExtensionTable.Builder extensionTableBuilder =
ReadRel.ExtensionTable.newBuilder();
- extensionTableBuilder.setDetail(
-
Any.newBuilder().setValue(ByteString.copyFromUtf8(extensionTableStr.toString())));
- return extensionTableBuilder.build();
- }
-}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java
b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java
deleted file mode 100644
index c6afa6578..000000000
---
a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.substrait.ddlplan;
-
-import org.apache.gluten.substrait.SubstraitContext;
-import org.apache.gluten.substrait.plan.PlanNode;
-
-import io.substrait.proto.DllPlan;
-import io.substrait.proto.InsertPlan;
-
-import java.io.Serializable;
-
-public class InsertPlanNode implements DllPlanNode, Serializable {
-
- private final PlanNode inputNode;
-
- private final SubstraitContext context;
-
- public InsertPlanNode(SubstraitContext context, PlanNode inputNode) {
- this.inputNode = inputNode;
- this.context = context;
- }
-
- @Override
- public DllPlan toProtobuf() {
- InsertPlan.Builder insertBuilder = InsertPlan.newBuilder();
- insertBuilder.setInput(inputNode.toProtobuf());
- if (context.getInsertOutputNode() != null) {
- insertBuilder.setOutput(context.getInsertOutputNode().toProtobuf());
- }
-
- DllPlan.Builder dllBuilder = DllPlan.newBuilder();
- dllBuilder.setInsertPlan(insertBuilder.build());
- return dllBuilder.build();
- }
-}
diff --git a/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto
b/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto
deleted file mode 100644
index 833ec8736..000000000
--- a/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package substrait;
-
-import "substrait/plan.proto";
-import "substrait/algebra.proto";
-
-option java_multiple_files = true;
-option java_package = "io.substrait.proto";
-option csharp_namespace = "Substrait.Protobuf";
-
-message DllPlan {
- oneof dll_type {
- InsertPlan insert_plan = 1;
- }
-}
-
-message InsertPlan {
- Plan input = 1;
- ReadRel.ExtensionTable output = 2;
-}
-
-message Dll {
- repeated DllPlan dll_plan = 1;
-}
\ No newline at end of file
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala
b/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala
index 6cc35bd16..79148d9f3 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala
@@ -16,8 +16,6 @@
*/
package org.apache.gluten.substrait
-import org.apache.gluten.substrait.ddlplan.InsertOutputNode
-
import java.lang.{Long => JLong}
import java.security.InvalidParameterException
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList,
Map => JMap}
@@ -64,16 +62,9 @@ class SubstraitContext extends Serializable {
private val aggregationParamsMap = new JHashMap[JLong, AggregationParams]()
private var iteratorIndex: JLong = 0L
- private var insertOutputNode: InsertOutputNode = _
private var operatorId: JLong = 0L
private var relId: JLong = 0L
- def getInsertOutputNode: InsertOutputNode = insertOutputNode
-
- def setInsertOutputNode(insertOutputNode: InsertOutputNode): Unit = {
- this.insertOutputNode = insertOutputNode
- }
-
def registerFunction(funcName: String): JLong = {
if (!functionMap.containsKey(funcName)) {
val newFunctionId: JLong = functionMap.size.toLong
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
