This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 05e16b7f292a342cdffa22d489646c55a0be6b39 Author: Y Ethan Guo <[email protected]> AuthorDate: Sun Mar 3 22:09:12 2024 -0800 [HUDI-7471] Use existing util method to get Spark conf in tests (#10802) --- .../org/apache/hudi/testutils/HoodieClientTestUtils.java | 2 +- .../org/apache/hudi/testutils/providers/SparkProvider.java | 2 +- .../execution/datasources/TestHoodieInMemoryFileIndex.scala | 5 ++--- .../scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala | 9 +++++++-- .../test/scala/org/apache/hudi/TestHoodieSparkUtils.scala | 13 +++---------- .../utilities/deltastreamer/TestSourceFormatAdapter.java | 5 ++--- .../utilities/sources/helpers/TestSanitizationUtils.java | 6 ++---- .../apache/hudi/utilities/testutils/UtilitiesTestBase.java | 2 +- .../utilities/transform/TestSqlQueryBasedTransformer.java | 4 ++-- 9 files changed, 21 insertions(+), 27 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java index 2413bf2dffd..57a2793f0f6 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java @@ -87,7 +87,7 @@ public class HoodieClientTestUtils { */ public static SparkConf getSparkConfForTest(String appName) { SparkConf sparkConf = new SparkConf().setAppName(appName) - .setMaster("local[4]") + .setMaster("local[8]") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "org.apache.spark.HoodieSparkKryoRegistrar") .set("spark.sql.shuffle.partitions", "4") diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java index 3a8bb1a300f..91045034e5f 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java @@ -38,7 +38,7 @@ public interface SparkProvider extends org.apache.hudi.testutils.providers.Hoodi default SparkConf conf(Map<String, String> overwritingConfigs) { SparkConf sparkConf = new SparkConf(); sparkConf.set("spark.app.name", getClass().getName()); - sparkConf.set("spark.master", "local[*]"); + sparkConf.set("spark.master", "local[8]"); sparkConf.set("spark.default.parallelism", "4"); sparkConf.set("spark.sql.shuffle.partitions", "4"); sparkConf.set("spark.driver.maxResultSize", "2g"); diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala index 8e7f6bf14b7..c9052a952e6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala @@ -18,6 +18,7 @@ package org.apache.spark.execution.datasources import org.apache.hadoop.fs.Path +import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest import org.apache.spark.sql.SparkSession import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -31,9 +32,7 @@ class TestHoodieInMemoryFileIndex { @Test def testCreateInMemoryIndex(@TempDir tempDir: File): Unit = { val spark = SparkSession.builder - .appName("Hoodie Datasource test") - .master("local[2]") - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(getSparkConfForTest("Hoodie Datasource test")) .getOrCreate val folders: Seq[Path] = Seq( diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala index c57785e5ffe..d7a1f9331ae 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala @@ -243,10 +243,15 @@ class TestHoodieSparkSqlWriter { @Test def testThrowExceptionInvalidSerializer(): Unit = { spark.stop() - val session = SparkSession.builder().appName("hoodie_test").master("local").getOrCreate() + val session = SparkSession.builder() + // Here we intentionally remove the "spark.serializer" config to test failure + .config(getSparkConfForTest("hoodie_test").remove("spark.serializer")) + .getOrCreate() try { val sqlContext = session.sqlContext - val options = Map("path" -> "hoodie/test/path", HoodieWriteConfig.TBL_NAME.key -> "hoodie_test_tbl") + val options = Map( + "path" -> (tempPath.toUri.toString + "/testThrowExceptionInvalidSerializer/basePath"), + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test_tbl") val e = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.ErrorIfExists, options, session.emptyDataFrame)) assert(e.getMessage.contains("spark.serializer")) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala index 15b6b2b35da..85c3c619111 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala @@ -20,6 +20,7 @@ package org.apache.hudi import org.apache.avro.generic.GenericRecord import org.apache.hudi.testutils.DataSourceTestUtils +import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest import org.apache.spark.sql.types.{ArrayType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.junit.jupiter.api.Assertions._ @@ -88,11 +89,7 @@ class TestHoodieSparkUtils { @Test def testCreateRddSchemaEvol(): Unit = { val spark = SparkSession.builder - .appName("Hoodie Datasource test") - .master("local[2]") - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - .config("spark.kryo.registrator", "org.apache.spark.HoodieSparkKryoRegistrar") - .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") + .config(getSparkConfForTest("Hoodie Datasource test")) .getOrCreate val schema = DataSourceTestUtils.getStructTypeExampleSchema @@ -126,11 +123,7 @@ class TestHoodieSparkUtils { @Test def testCreateRddWithNestedSchemas(): Unit = { val spark = SparkSession.builder - .appName("Hoodie Datasource test") - .master("local[2]") - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - .config("spark.kryo.registrator", "org.apache.spark.HoodieSparkKryoRegistrar") - .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") + .config(getSparkConfForTest("Hoodie Datasource test")) .getOrCreate val innerStruct1 = new StructType().add("innerKey","string",false).add("innerValue", "long", true) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java index 1d6f2f110b2..788105c2028 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java @@ -49,6 +49,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.util.stream.Stream; +import static org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -64,9 +65,7 @@ public class TestSourceFormatAdapter { public static void start() { spark = SparkSession .builder() - .master("local[*]") - .appName(TestSourceFormatAdapter.class.getName()) - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(getSparkConfForTest(TestSourceFormatAdapter.class.getName())) .getOrCreate(); jsc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java index 1a660ac7135..39dfa430268 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java @@ -22,7 +22,6 @@ package org.apache.hudi.utilities.sources.helpers; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; -import org.apache.hudi.utilities.deltastreamer.TestSourceFormatAdapter; import org.apache.hudi.utilities.testutils.SanitizationTestUtils; import org.apache.avro.Schema; @@ -45,6 +44,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.stream.Stream; +import static org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest; import static org.apache.hudi.utilities.testutils.SanitizationTestUtils.generateProperFormattedSchema; import static org.apache.hudi.utilities.testutils.SanitizationTestUtils.generateRenamedSchemaWithConfiguredReplacement; import static org.apache.hudi.utilities.testutils.SanitizationTestUtils.generateRenamedSchemaWithDefaultReplacement; @@ -61,9 +61,7 @@ public class TestSanitizationUtils { public static void start() { spark = SparkSession .builder() - .master("local[*]") - .appName(TestSourceFormatAdapter.class.getName()) - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(getSparkConfForTest(TestSanitizationUtils.class.getName())) .getOrCreate(); jsc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index f68d88253e2..298a76a2aff 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -159,7 +159,7 @@ public class UtilitiesTestBase { zookeeperTestService.start(); } - jsc = UtilHelpers.buildSparkContext(UtilitiesTestBase.class.getName() + "-hoodie", "local[4]"); + jsc = UtilHelpers.buildSparkContext(UtilitiesTestBase.class.getName() + "-hoodie", "local[8]"); context = new HoodieSparkEngineContext(jsc); sqlContext = new SQLContext(jsc); sparkSession = SparkSession.builder().config(jsc.getConf()).getOrCreate(); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java index b6fdc258242..6f05dc1b184 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestSqlQueryBasedTransformer.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test; import java.util.Collections; +import static org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -39,8 +40,7 @@ public class TestSqlQueryBasedTransformer { SparkSession spark = SparkSession .builder() - .master("local[2]") - .appName(TestSqlQueryBasedTransformer.class.getName()) + .config(getSparkConfForTest(TestSqlQueryBasedTransformer.class.getName())) .getOrCreate(); JavaSparkContext jsc = JavaSparkContext.fromSparkContext(spark.sparkContext());
