This is an automated email from the ASF dual-hosted git repository.
garyli pushed a commit to branch release-0.8.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/release-0.8.0 by this push:
new 3996044 [HOTFIX] close spark session in functional test suite and
disable spark3 test for spark2 (#2727)
3996044 is described below
commit 399604412b9a5f59ae88a0dac98d94e61638a43a
Author: Gary Li <yanjia.gary...@gmail.com>
AuthorDate: Fri Mar 26 20:58:29 2021 -0700
[HOTFIX] close spark session in functional test suite and disable spark3
test for spark2 (#2727)
---
.../org/apache/hudi/testutils/FunctionalTestHarness.java | 16 ++++++++++++++++
hudi-spark-datasource/hudi-spark2/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark3/pom.xml | 7 +++++++
.../hudi/utilities/testutils/UtilitiesTestBase.java | 9 +++++++++
pom.xml | 2 ++
5 files changed, 35 insertions(+), 1 deletion(-)
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
index fc02e6d..e391abf 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
@@ -152,6 +152,8 @@ public class FunctionalTestHarness implements
SparkProvider, DFSProvider, Hoodie
hdfsTestService.stop();
hdfsTestService = null;
+ jsc.close();
+ jsc = null;
spark.stop();
spark = null;
}));
@@ -166,5 +168,19 @@ public class FunctionalTestHarness implements
SparkProvider, DFSProvider, Hoodie
for (FileStatus f : fileStatuses) {
fs.delete(f.getPath(), true);
}
+ if (hdfsTestService != null) {
+ hdfsTestService.stop();
+ hdfsTestService = null;
+ }
+ if (spark != null) {
+ spark.stop();
+ spark = null;
+ }
+ if (jsc != null) {
+ jsc.close();
+ jsc = null;
+ }
+ sqlContext = null;
+ context = null;
}
}
diff --git a/hudi-spark-datasource/hudi-spark2/pom.xml
b/hudi-spark-datasource/hudi-spark2/pom.xml
index 91718ab..91f4849 100644
--- a/hudi-spark-datasource/hudi-spark2/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2/pom.xml
@@ -151,7 +151,7 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>${scala11.version}</version>
+ <version>${scala.version}</version>
</dependency>
<!-- Hoodie -->
diff --git a/hudi-spark-datasource/hudi-spark3/pom.xml
b/hudi-spark-datasource/hudi-spark3/pom.xml
index d6028bb..813c4e7 100644
--- a/hudi-spark-datasource/hudi-spark3/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3/pom.xml
@@ -125,6 +125,13 @@
</configuration>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>${skip.hudi-spark3.unit.tests}</skipTests>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
index 6efd468..0adef52 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
@@ -124,15 +124,19 @@ public class UtilitiesTestBase {
public static void cleanupClass() {
if (hdfsTestService != null) {
hdfsTestService.stop();
+ hdfsTestService = null;
}
if (hiveServer != null) {
hiveServer.stop();
+ hiveServer = null;
}
if (hiveTestService != null) {
hiveTestService.stop();
+ hiveTestService = null;
}
if (zookeeperTestService != null) {
zookeeperTestService.stop();
+ zookeeperTestService = null;
}
}
@@ -150,6 +154,11 @@ public class UtilitiesTestBase {
TestDataSource.resetDataGen();
if (jsc != null) {
jsc.stop();
+ jsc = null;
+ }
+ if (sparkSession != null) {
+ sparkSession.close();
+ sparkSession = null;
}
if (context != null) {
context = null;
diff --git a/pom.xml b/pom.xml
index 8ee6831..db1a798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,6 +133,7 @@
<skipFTs>${skipTests}</skipFTs>
<skipITs>${skipTests}</skipITs>
<skip.hudi-spark2.unit.tests>${skipTests}</skip.hudi-spark2.unit.tests>
+ <skip.hudi-spark3.unit.tests>${skipTests}</skip.hudi-spark3.unit.tests>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<main.basedir>${project.basedir}</main.basedir>
<spark.bundle.hive.scope>provided</spark.bundle.hive.scope>
@@ -1424,6 +1425,7 @@
<properties>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
+ <skip.hudi-spark3.unit.tests>true</skip.hudi-spark3.unit.tests>
</properties>
<activation>
<property>
