This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.14.1-spark35-scala213 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit af38c325f6495a468542dd8627f06cf094f7a480 Author: Y Ethan Guo <[email protected]> AuthorDate: Wed May 1 22:21:00 2024 -0700 [HUDI-7702] Remove unused method in ReflectUtil (#11135) --- .../apache/hudi/spark3/internal/ReflectUtil.java | 29 +----------- .../hudi/spark3/internal/TestReflectUtil.java | 54 ---------------------- .../hudi/spark3/internal/TestReflectUtil.java | 54 ---------------------- .../hudi/spark3/internal/TestReflectUtil.java | 54 ---------------------- 4 files changed, 1 insertion(+), 190 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java index ad83720b021..c726777876f 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java @@ -18,41 +18,14 @@ package org.apache.hudi.spark3.internal; import org.apache.hudi.HoodieSparkUtils; -import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; -import org.apache.spark.sql.catalyst.util.DateFormatter; -import scala.Option; -import scala.collection.Seq; -import scala.collection.immutable.Map; +import org.apache.spark.sql.catalyst.util.DateFormatter; -import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.time.ZoneId; public class ReflectUtil { - public static InsertIntoStatement createInsertInto(LogicalPlan table, Map<String, Option<String>> partition, Seq<String> userSpecifiedCols, - LogicalPlan query, boolean overwrite, boolean ifPartitionNotExists, boolean byName) { - try { - if (HoodieSparkUtils.gteqSpark3_5()) { - Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor( - LogicalPlan.class, Map.class, Seq.class, LogicalPlan.class, boolean.class, boolean.class, boolean.class); - return constructor.newInstance(table, partition, userSpecifiedCols, query, overwrite, ifPartitionNotExists, byName); - } else if (HoodieSparkUtils.isSpark3_0()) { - Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor( - LogicalPlan.class, Map.class, LogicalPlan.class, boolean.class, boolean.class); - return constructor.newInstance(table, partition, query, overwrite, ifPartitionNotExists); - } else { - Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor( - LogicalPlan.class, Map.class, Seq.class, LogicalPlan.class, boolean.class, boolean.class); - return constructor.newInstance(table, partition, userSpecifiedCols, query, overwrite, ifPartitionNotExists); - } - } catch (Exception e) { - throw new RuntimeException("Error in create InsertIntoStatement", e); - } - } - public static DateFormatter getDateFormatter(ZoneId zoneId) { try { ClassLoader loader = Thread.currentThread().getContextClassLoader(); diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java deleted file mode 100644 index 0763a22f032..00000000000 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.spark3.internal; - -import org.apache.hudi.testutils.HoodieClientTestBase; - -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; -import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Unit tests {@link ReflectUtil}. - */ -public class TestReflectUtil extends HoodieClientTestBase { - - @Test - public void testDataSourceWriterExtraCommitMetadata() throws Exception { - SparkSession spark = sqlContext.sparkSession(); - - String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')"; - InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql); - - InsertIntoStatement newStatment = ReflectUtil.createInsertInto( - statement.table(), - statement.partitionSpec(), - scala.collection.immutable.List.empty(), - statement.query(), - statement.overwrite(), - statement.ifPartitionNotExists(), - false); - - Assertions.assertTrue( - ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util")); - } -} diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java deleted file mode 100644 index 0763a22f032..00000000000 --- a/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.spark3.internal; - -import org.apache.hudi.testutils.HoodieClientTestBase; - -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; -import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Unit tests {@link ReflectUtil}. - */ -public class TestReflectUtil extends HoodieClientTestBase { - - @Test - public void testDataSourceWriterExtraCommitMetadata() throws Exception { - SparkSession spark = sqlContext.sparkSession(); - - String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')"; - InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql); - - InsertIntoStatement newStatment = ReflectUtil.createInsertInto( - statement.table(), - statement.partitionSpec(), - scala.collection.immutable.List.empty(), - statement.query(), - statement.overwrite(), - statement.ifPartitionNotExists(), - false); - - Assertions.assertTrue( - ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util")); - } -} diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java deleted file mode 100644 index 5a08e54f5e1..00000000000 --- a/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.spark3.internal; - -import org.apache.hudi.testutils.HoodieClientTestBase; - -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; -import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Unit tests {@link ReflectUtil}. - */ -public class TestReflectUtil extends HoodieClientTestBase { - - @Test - public void testDataSourceWriterExtraCommitMetadata() throws Exception { - SparkSession spark = sqlContext.sparkSession(); - - String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')"; - InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql); - - InsertIntoStatement newStatment = ReflectUtil.createInsertInto( - statement.table(), - statement.partitionSpec(), - scala.collection.immutable.List.empty(), - statement.query(), - statement.overwrite(), - statement.ifPartitionNotExists(), - statement.byName()); - - Assertions.assertTrue( - ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util")); - } -}
