This is an automated email from the ASF dual-hosted git repository.
vhs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 894b817f79af chore(ci): cleanup for print statements, showing
tables/schemas (#17771)
894b817f79af is described below
commit 894b817f79af42fa61179ed9d0842d61e7d3c07c
Author: Tim Brown <[email protected]>
AuthorDate: Tue Feb 24 00:11:40 2026 -0800
chore(ci): cleanup for print statements, showing tables/schemas (#17771)
* cleanup for print statements, showing tables/schemas, direct GC calls
* simplify test resource cleanup for spark tests
* fix scalastyle
* Revert "simplify test resource cleanup for spark tests"
This reverts commit fde72c1e60f2801cd7f82ef92c7c0d6b0adc2bb8.
* minimize diff
* add back gc call to see if there is impact on runtime
* limit stacktrace, set jetty to warn
* update logging, add assertNotNull on schema
---
.github/workflows/bot.yml | 2 +-
azure-pipelines-20230430.yml | 2 +-
.../hudi/cli/commands/TestLockAuditingCommand.java | 15 +-
.../testutils/HoodieCLIIntegrationTestBase.java | 1 -
.../testutils/HoodieCLIIntegrationTestHarness.java | 6 -
.../client/utils/TestDeletePartitionUtils.java | 2 -
.../testutils/HoodieFlinkClientTestHarness.java | 1 -
.../hudi/execution/TestBoundedInMemoryQueue.java | 2 +-
.../TestClosableSortedDedupingIterator.java | 16 +-
.../TestHoodieNativeAvroHFileReaderCaching.java | 39 ++--
.../hudi/hadoop/utils/TestHiveAvroSerializer.java | 1 -
.../apache/hudi/TestDecimalTypeDataWorkflow.scala | 6 +-
...DataValidationCheckForLogCompactionActions.java | 16 --
.../TestRemoteFileSystemViewWithMetadataTable.java | 1 -
...SparkBinaryCopyClusteringAndValidationMeta.java | 2 +-
.../TestIncrementalQueryWithArchivedInstants.scala | 2 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 204 ++++++++++-----------
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 6 +-
.../org/apache/hudi/TestInsertDedupPolicy.scala | 2 -
.../functional/PartitionStatsIndexTestBase.scala | 1 -
.../TestAutoGenerationOfRecordKeys.scala | 8 -
.../hudi/functional/TestBasicSchemaEvolution.scala | 8 -
.../apache/hudi/functional/TestCOWDataSource.scala | 5 +-
.../apache/hudi/functional/TestMORDataSource.scala | 12 +-
.../functional/TestPartialUpdateAvroPayload.scala | 8 -
.../TestSparkDataSourceDAGExecution.scala | 9 -
.../functional/cdc/TestCDCDataFrameSuite.scala | 6 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 4 +-
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 13 +-
.../sql/hudi/feature/TestCDCForSparkSQL.scala | 28 +--
.../resources/log4j2-surefire-quiet.properties | 2 +-
.../src/main/resources/log4j2-surefire.properties | 2 +-
32 files changed, 176 insertions(+), 256 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 57833a9e54b1..81279430e77a 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -28,7 +28,7 @@ concurrency:
cancel-in-progress: ${{ !contains(github.ref, 'master') &&
!contains(github.ref, 'branch-0.x') && !contains(github.ref, 'release-') }}
env:
- MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5
+ MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5 -Dorg.eclipse.jetty.LEVEL=WARN
SPARK_COMMON_MODULES:
hudi-spark-datasource/hudi-spark,hudi-spark-datasource/hudi-spark-common
JAVA_UT_FILTER1:
-Dtest=!TestCOWDataSource,!TestMORDataSource,!TestHoodieFileSystemViews
JAVA_UT_FILTER2:
-Dtest=TestCOWDataSource,TestMORDataSource,TestHoodieFileSystemViews
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index a458f9f6a2db..a6ed775b2c87 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -102,7 +102,7 @@ parameters:
variables:
BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink1.18'
PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -ntp -B -V -Pwarn-log
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
- MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests
$(BUILD_PROFILES) $(PLUGIN_OPTS)
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5'
+ MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests
$(BUILD_PROFILES) $(PLUGIN_OPTS)
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5 -Dorg.eclipse.jetty.LEVEL=WARN'
MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
JAVA_MVN_TEST_FILTER: '-DwildcardSuites=skipScalaTests
-Dsurefire.failIfNoSpecifiedTests=false'
SCALA_MVN_TEST_FILTER: '-Dtest=skipJavaTests
-Dsurefire.failIfNoSpecifiedTests=false'
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestLockAuditingCommand.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestLockAuditingCommand.java
index 14d6d9d2e9c0..52f9c9fc852f 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestLockAuditingCommand.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestLockAuditingCommand.java
@@ -22,8 +22,8 @@ import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
import
org.apache.hudi.client.transaction.lock.audit.StorageLockProviderAuditService;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.io.util.FileIOUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -31,17 +31,17 @@ import org.apache.hudi.storage.StoragePathInfo;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.shell.Shell;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -888,8 +888,7 @@ public class TestLockAuditingCommand extends
CLIFunctionalTestHarness {
createAuditFiles(scenarios);
Object result = shell.evaluate(() -> "locks audit cleanup --dryRun true");
- System.out.println("DEBUG - Mixed files result: " + result.toString());
-
+
assertAll("Multiple files handled correctly",
() -> assertTrue(ShellEvaluationResultUtil.isSuccess(result)),
() -> assertNotNull(result.toString()),
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
index 9d75bd17c3d6..d15ed0dbc159 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
@@ -34,7 +34,6 @@ public class HoodieCLIIntegrationTestBase extends
HoodieCLIIntegrationTestHarnes
initResources();
}
- @Override
@AfterEach
public void teardown() throws Exception {
cleanupResources();
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java
index 2e0271a07ddc..37275d8b1518 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java
@@ -20,7 +20,6 @@ package org.apache.hudi.cli.testutils;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
/**
@@ -33,11 +32,6 @@ public class HoodieCLIIntegrationTestHarness extends
HoodieSparkClientTestHarnes
initPath();
}
- @AfterEach
- public void teardown() throws Exception {
- System.gc();
- }
-
/**
* Helper to prepare string for matching.
* @param str Input string.
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
index e55d9e614f62..bda926e0bf3e 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
@@ -69,8 +69,6 @@ public class TestDeletePartitionUtils {
boolean hasPendingCompactionOperations,
boolean hasPendingLogCompactionOperations,
boolean hasFileGroupsInPendingClustering) {
- System.out.printf("hasPendingCompactionOperations: %s,
hasPendingLogCompactionOperations: %s, hasFileGroupsInPendingClustering: %s%n",
- hasPendingCompactionOperations, hasPendingLogCompactionOperations,
hasFileGroupsInPendingClustering);
Mockito.when(table.getSliceView()).thenReturn(fileSystemView);
Mockito.when(fileSystemView.getPendingCompactionOperations()).thenReturn(createPendingCompactionOperations(hasPendingCompactionOperations));
Mockito.when(fileSystemView.getPendingLogCompactionOperations()).thenReturn(createPendingCompactionOperations(hasPendingLogCompactionOperations));
diff --git
a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java
b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java
index 60a06c8a0970..e6e37b7e375a 100644
---
a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java
+++
b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java
@@ -119,7 +119,6 @@ public class HoodieFlinkClientTestHarness extends
HoodieCommonTestHarness {
cleanupTestDataGenerator();
cleanupFileSystem();
cleanupExecutorService();
- System.gc();
}
/**
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java
index 0fff4bf752a1..4affa73e4fa3 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java
@@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.util.DefaultSizeEstimator;
+import org.apache.hudi.io.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.SizeEstimator;
import org.apache.hudi.common.util.queue.BoundedInMemoryQueue;
@@ -31,7 +32,6 @@ import org.apache.hudi.common.util.queue.HoodieProducer;
import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.io.util.FileIOUtils;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.avro.generic.IndexedRecord;
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestClosableSortedDedupingIterator.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestClosableSortedDedupingIterator.java
index 7e98d30f65fc..bbb3eea66dcc 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestClosableSortedDedupingIterator.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestClosableSortedDedupingIterator.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.util.collection;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
@@ -33,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+@Slf4j
public class TestClosableSortedDedupingIterator {
@Test
@@ -231,32 +233,32 @@ public class TestClosableSortedDedupingIterator {
List<String> elementsWithConsecutiveNulls = Arrays.asList("a", null, null,
"b", null, null);
ClosableSortedDedupingIterator<String> dedupingIterator =
new
ClosableSortedDedupingIterator<>(elementsWithConsecutiveNulls.iterator());
-
+
// Debug: trace through each step
- System.out.println("Input: " + elementsWithConsecutiveNulls);
-
+ log.debug("Input: {}", elementsWithConsecutiveNulls);
+
// First element
assertTrue(dedupingIterator.hasNext());
String first = dedupingIterator.next();
- System.out.println("1. " + first);
+ log.debug("1. {}", first);
assertEquals("a", first);
// Second element
assertTrue(dedupingIterator.hasNext());
String second = dedupingIterator.next();
- System.out.println("2. " + second);
+ log.debug("2. {}", second);
assertNull(second);
// Third element
assertTrue(dedupingIterator.hasNext());
String third = dedupingIterator.next();
- System.out.println("3. " + third);
+ log.debug("3. {}", third);
assertEquals("b", third);
// Fourth element
assertTrue(dedupingIterator.hasNext());
String fourth = dedupingIterator.next();
- System.out.println("4. " + fourth);
+ log.debug("4. {}", fourth);
assertNull(fourth);
// No more elements
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
index 227fa629b17f..fca1549a9965 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
@@ -31,11 +31,12 @@ import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
-import org.apache.hudi.io.storage.hadoop.HoodieAvroHFileWriter;
import org.apache.hudi.io.hadoop.TestHoodieOrcReaderWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroHFileWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
+import lombok.extern.slf4j.Slf4j;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
@@ -59,8 +60,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+@Slf4j
public class TestHoodieNativeAvroHFileReaderCaching {
-
@TempDir
public static Path tempDir;
@@ -79,7 +80,7 @@ public class TestHoodieNativeAvroHFileReaderCaching {
// Write records with for realistic testing
final int numRecords = 50_000;
- System.out.println("Creating HFile with " + numRecords + " records...");
+ log.debug("Creating HFile with {} records...", numRecords);
for (int i = 0; i < numRecords; i++) {
String key = String.format("key_%08d", i);
@@ -102,14 +103,14 @@ public class TestHoodieNativeAvroHFileReaderCaching {
MISSING_KEYS.add(missingKey);
}
- System.out.println("HFile created with " + EXISTING_KEYS.size() + "
existing keys");
- System.out.println("Generated " + MISSING_KEYS.size() + " missing keys for
testing");
+ log.debug("HFile created with {} existing keys", EXISTING_KEYS.size());
+ log.debug("Generated {} missing keys for testing", MISSING_KEYS.size());
}
@Test
@Disabled("Enable this for local performance tests")
public void testBlockCachePerformanceOnRecordLevelIndex() throws Exception {
- System.out.println("\n=== HFile BlockCache Performance Test ===");
+ log.debug("\n=== HFile BlockCache Performance Test ===");
// Test existing keys lookup performance
testExistingKeysLookup();
@@ -117,11 +118,11 @@ public class TestHoodieNativeAvroHFileReaderCaching {
// Test missing keys lookup performance
testMissingKeysLookup();
-
System.out.println("================================================================\n");
+
log.debug("================================================================\n");
}
private void testExistingKeysLookup() throws Exception {
- System.out.println("\n--- Testing " + KEYS_TO_LOOKUP + " Existing Key
Lookups ---");
+ log.debug("\n--- Testing {} Existing Key Lookups ---", KEYS_TO_LOOKUP);
// Select 10K random existing keys
Collections.shuffle(EXISTING_KEYS, RANDOM);
@@ -137,17 +138,17 @@ public class TestHoodieNativeAvroHFileReaderCaching {
double speedup = (double) noCacheTime / cacheTime;
- System.out.printf(KEYS_TO_LOOKUP + " Existing Key Lookups:\n");
- System.out.printf(" - Without BlockCache: %d ms\n", noCacheTime);
- System.out.printf(" - With BlockCache: %d ms\n", cacheTime);
- System.out.printf(" - Speedup: %.2fx\n", speedup);
- System.out.printf(" - Performance Improvement: %.1f%%\n", (speedup - 1) *
100);
+ log.debug("{} Existing Key Lookups:\n"
+ + " - Without BlockCache: {} ms\n"
+ + " - With BlockCache: {} ms\n"
+ + " - Speedup: {}x\n"
+ + " - Performance Improvement: {}%\n", KEYS_TO_LOOKUP, noCacheTime,
cacheTime, speedup, (speedup - 1) * 100);
assertTrue(speedup > 1.0, "BlockCache should provide speedup for existing
key lookups");
}
private void testMissingKeysLookup() throws Exception {
- System.out.println("\n--- Testing " + KEYS_TO_LOOKUP + " Missing Key
Lookups ---");
+ log.debug("\n--- Testing {} Missing Key Lookups ---", KEYS_TO_LOOKUP);
// Use all 1k missing keys
List<String> testKeys = new ArrayList<>(MISSING_KEYS);
@@ -163,11 +164,11 @@ public class TestHoodieNativeAvroHFileReaderCaching {
double speedup = (double) noCacheTime / cacheTime;
- System.out.printf(KEYS_TO_LOOKUP + " Missing Key Lookups:\n");
- System.out.printf(" - Without BlockCache: %d ms\n", noCacheTime);
- System.out.printf(" - With BlockCache: %d ms\n", cacheTime);
- System.out.printf(" - Speedup: %.2fx\n", speedup);
- System.out.printf(" - Performance Improvement: %.1f%%\n", (speedup - 1) *
100);
+ log.debug("{} Existing Key Lookups:\n"
+ + " - Without BlockCache: {} ms\n"
+ + " - With BlockCache: {} ms\n"
+ + " - Speedup: {}x\n"
+ + " - Performance Improvement: {}%\n", KEYS_TO_LOOKUP, noCacheTime,
cacheTime, speedup, (speedup - 1) * 100);
// Missing keys may not benefit as much from caching but should not be
slower
assertTrue(speedup >= 0.8, "BlockCache should not significantly slow down
missing key lookups");
diff --git
a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java
b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java
index d4dac0934a25..d4dd501669e9 100644
---
a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java
+++
b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java
@@ -588,7 +588,6 @@ public class TestHiveAvroSerializer {
HoodieSchema schema = HoodieSchema.parse(schemaWithTimeMillis);
List<TypeInfo> columnTypes = HiveTypeUtils.generateColumnTypes(schema);
- System.out.println(columnTypes);
assertEquals(TypeInfoFactory.intTypeInfo, columnTypes.get(0));
// The second column should be int type
assertEquals(TypeInfoFactory.intTypeInfo, columnTypes.get(1));
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDecimalTypeDataWorkflow.scala
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDecimalTypeDataWorkflow.scala
index fee69a855c20..b4cb31f31a15 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDecimalTypeDataWorkflow.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDecimalTypeDataWorkflow.scala
@@ -121,10 +121,8 @@ class TestDecimalTypeDataWorkflow extends
SparkClientFunctionalTestHarness{
.toDF("id", "decimal_col").sort("id")
val expectedMinusActual = expectedDf.except(actual)
val actualMinusExpected = actual.except(expectedDf)
- expectedDf.show(false)
- actual.show(false)
- expectedMinusActual.show(false)
- actualMinusExpected.show(false)
+ expectedDf.collect()
+ actual.collect()
assertTrue(expectedMinusActual.isEmpty && actualMinusExpected.isEmpty)
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java
index 73568ebb48f2..bb6c04a781f1 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java
@@ -50,7 +50,6 @@ import org.apache.hudi.testutils.MetadataMergeWriteStatus;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.spark.api.java.JavaRDD;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
@@ -100,21 +99,6 @@ public class TestDataValidationCheckForLogCompactionActions
extends HoodieClient
HoodieSparkWriteableTestTable.of(metaClient);
}
- //TODO: include both the table's contents.
- /**
- * Cleanups resource group for the subclasses of {@link
HoodieClientTestBase}.
- */
- @AfterEach
- public void cleanupResources() throws IOException {
- cleanupTimelineService();
- cleanupClients();
- cleanupSparkContexts();
- cleanupTestDataGenerator();
- cleanupFileSystem();
- cleanupExecutorService();
- System.gc();
- }
-
/**
* Stress test logcompaction along with compaction by following approach.
* a. Create a random seed to do insert/upsert/deleting operations on main
table and replicate same action on experiment table.
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
index 28ed9741d744..1d48ea1aea56 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -94,7 +94,6 @@ public class TestRemoteFileSystemViewWithMetadataTable
extends HoodieSparkClient
cleanupFileSystem();
cleanupExecutorService();
dataGen = null;
- System.gc();
}
@Override
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
index e3cf43fd8a7b..ea8e40b72e13 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
@@ -27,6 +27,7 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
+import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.engine.LocalTaskContextSupplier;
@@ -35,7 +36,6 @@ import org.apache.hudi.common.model.ClusteringOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.HoodieTableType;
-import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala
index c538a75a742d..b57ad0854cd1 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala
@@ -100,7 +100,7 @@ class TestIncrementalQueryWithArchivedInstants extends
SparkClientFunctionalTest
.option("hoodie.datasource.read.begin.instanttime", "0")
.options(opt)
.load(tblPath)
- .show(false)
+ .collect()
}
})
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
index 56eccc0a1376..bd53af1997f5 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
@@ -27,7 +27,7 @@ import org.apache.hudi.testutils.HoodieClientTestBase
import org.apache.spark.SparkException
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.sql.types._
-import org.junit.jupiter.api.{AfterEach, BeforeEach}
+import org.junit.jupiter.api.{AfterEach, Assertions, BeforeEach}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{CsvSource, ValueSource}
@@ -119,8 +119,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
try {
// convert int to string first before conversion to binary
val initDF = prepDataFrame(df1, colInitType)
- initDF.printSchema()
- initDF.show(false)
+ Assertions.assertNotNull(initDF.schema.toString())
+ initDF.collect
// recreate table
initialiseTable(initDF, tempRecordPath, tableType)
@@ -128,8 +128,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// perform avro supported casting
var upsertDf = prepDataFrame(df2, colInitType)
upsertDf = castColToX(a, colToCast, upsertDf)
- upsertDf.printSchema()
- upsertDf.show(false)
+ Assertions.assertNotNull(upsertDf.schema.toString())
+ upsertDf.collect
// upsert
upsertData(upsertDf, tempRecordPath, tableType)
@@ -137,9 +137,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ Assertions.assertNotNull(readDf.schema.toString())
+ readDf.collect
assert(true)
} catch {
@@ -177,16 +176,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// convert int to string first before conversion to binary
val initDF = df1
- initDF.printSchema()
- initDF.show(false)
+ initDF.schema.toString()
+ initDF.collect
// recreate table
initialiseTable(initDF, tempRecordPath, tableType)
// perform avro supported operation of adding a new column at the end of
the table
val upsertDf = df2
- upsertDf.printSchema()
- upsertDf.show(false)
+ upsertDf.schema.toString()
+ upsertDf.collect
// upsert
upsertData(upsertDf, tempRecordPath, tableType)
@@ -194,9 +193,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -217,16 +215,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// convert int to string first before conversion to binary
val initDF = df1
- initDF.printSchema()
- initDF.show(false)
+ initDF.schema.toString()
+ initDF.collect
// recreate table
initialiseTable(initDF, tempRecordPath, tableType)
// perform avro supported operation of deleting a column
val upsertDf = df2
- upsertDf.printSchema()
- upsertDf.show(false)
+ upsertDf.schema.toString()
+ upsertDf.collect
// upsert
assertThrows(classOf[SchemaCompatibilityException]) {
@@ -240,9 +238,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
//same param since that will only affect the reader
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -258,16 +255,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// convert int to string first before conversion to binary
val initDF = df1
- initDF.printSchema()
- initDF.show(false)
+ initDF.schema.toString()
+ initDF.collect
// recreate table
initialiseTable(initDF, tempRecordPath, tableType)
// perform avro supported operation of deleting a column
val upsertDf = df2
- upsertDf.printSchema()
- upsertDf.show(false)
+ upsertDf.schema.toString()
+ upsertDf.collect
// upsert
upsertData(upsertDf, tempRecordPath, tableType)
@@ -275,9 +272,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -297,8 +293,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
.add("pages", IntegerType)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayStructData),
arrayStructSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -318,17 +314,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayStructData),
newArrayStructSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -348,8 +343,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
.add("pages", IntegerType)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayStructData),
arrayStructSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -367,8 +362,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
.add("pages", LongType)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayStructData),
newArrayStructSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
@@ -376,9 +371,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
}
@@ -399,8 +393,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
.add("pages", IntegerType)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayStructData),
arrayStructSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -418,17 +412,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
.add("author", StringType)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayStructData),
newArrayStructSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -446,8 +439,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, IntegerType, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -463,8 +456,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, LongType, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
@@ -472,9 +465,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
}
@@ -502,8 +494,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, innerStructSchema, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -528,8 +520,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, newInnerStructSchema, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
@@ -537,9 +529,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
}
@@ -567,8 +558,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, innerStructSchema, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -594,17 +585,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, newInnerStructSchema, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -631,8 +621,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, innerStructSchema, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -657,17 +647,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, newInnerStructSchema, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -695,8 +684,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, innerStructSchema, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -720,17 +709,16 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, newInnerStructSchema, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType, true)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -751,42 +739,42 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// 1. Initialise table
val df1 = Seq((1, 100, newPartition)).toDF("id", "userid", "name")
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
initialiseTable(df1, tempRecordPath, tableType)
// 2. Promote INT type to LONG into a different partition
val df2 = Seq((2, 200L, newPartition)).toDF("id", "userid", "name")
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
upsertData(df2, tempRecordPath, tableType)
// 3. Promote LONG to FLOAT
var df3 = Seq((3, 300, newPartition)).toDF("id", "userid", "name")
df3 = df3.withColumn("userid", df3.col("userid").cast("float"))
- df3.printSchema()
- df3.show(false)
+ df3.schema.toString()
+ df3.collect
upsertData(df3, tempRecordPath)
// 4. Promote FLOAT to DOUBLE
var df4 = Seq((4, 400, newPartition)).toDF("id", "userid", "name")
df4 = df4.withColumn("userid", df4.col("userid").cast("float"))
- df4.printSchema()
- df4.show(false)
+ df4.schema.toString()
+ df4.collect
upsertData(df4, tempRecordPath)
// 5. Add two new column
var df5 = Seq((5, 500, "newcol1", "newcol2", newPartition)).toDF("id",
"userid", "newcol1", "newcol2", "name")
df5 = df5.withColumn("userid", df5.col("userid").cast("float"))
- df5.printSchema()
- df5.show(false)
+ df5.schema.toString()
+ df5.collect
upsertData(df5, tempRecordPath)
// 6. Delete a column
var df6 = Seq((6, 600, "newcol1", newPartition)).toDF("id", "userid",
"newcol1", "name")
df6 = df6.withColumn("userid", df6.col("userid").cast("float"))
- df6.printSchema()
- df6.show(false)
+ df6.schema.toString()
+ df6.collect
assertThrows(classOf[SchemaCompatibilityException]) {
upsertData(df6, tempRecordPath)
}
@@ -795,16 +783,15 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// 7. Rearrange column position
var df7 = Seq((7, "newcol1", 700, newPartition)).toDF("id", "newcol1",
"userid", "name")
df7 = df7.withColumn("userid", df7.col("userid").cast("float"))
- df7.printSchema()
- df7.show(false)
+ df7.schema.toString()
+ df7.collect
upsertData(df7, tempRecordPath)
// read out the table
val readDf = spark.read.format("hudi")
.load(tempRecordPath)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
@ParameterizedTest
@@ -822,8 +809,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, IntegerType, true)))
.add("name", StringType)
val df1 =
spark.createDataFrame(spark.sparkContext.parallelize(arrayMapData),
arrayMapSchema)
- df1.printSchema()
- df1.show(false)
+ df1.schema.toString()
+ df1.collect
// recreate table
initialiseTable(df1, tempRecordPath, tableType)
@@ -842,8 +829,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
new MapType(StringType, LongType, true)))
.add("name", StringType)
val df2 =
spark.createDataFrame(spark.sparkContext.parallelize(newArrayMapData),
newArrayMapSchema)
- df2.printSchema()
- df2.show(false)
+ df2.schema.toString()
+ df2.collect
// upsert
upsertData(df2, tempRecordPath, tableType)
@@ -870,9 +857,8 @@ class TestAvroSchemaResolutionSupport extends
HoodieClientTestBase with ScalaAss
// read out the table
val readDf = spark.read.format("hudi")
.load(path)
- readDf.printSchema()
- readDf.show(false)
- readDf.foreach(_ => {})
+ readDf.schema.toString()
+ readDf.collect
}
protected def withSQLConf[T](pairs: (String, String)*)(f: => T): T = {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 561e829dbe35..15f0b5d571b1 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -317,7 +317,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
// Ensure the partition column (i.e 'city') can be read back
val tripsDF = spark.read.format("hudi").load(tempBasePath)
- tripsDF.show()
+ tripsDF.collect()
tripsDF.select("city").foreach(row => {
assertNotNull(row)
})
@@ -329,7 +329,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
partitionPaths(i) = String.format("%s/%s/*", tempBasePath, partitions(i))
}
val rawFileDf = spark.sqlContext.read.parquet(partitionPaths(0),
partitionPaths(1), partitionPaths(2))
- rawFileDf.show()
+ rawFileDf.collect()
rawFileDf.select("city").foreach(row => {
assertNull(row.get(0))
})
@@ -939,7 +939,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
val recordsToDelete = spark.emptyDataFrame
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, fooTableModifier,
recordsToDelete)
val snapshotDF3 = spark.read.format("org.apache.hudi").load(tempBasePath)
- snapshotDF3.show()
+ snapshotDF3.count()
assertEquals(0, snapshotDF3.filter(entry => {
val partitionPath = entry.getString(3)
expectedPartitions.count(p => partitionPath.equals(p)) != 1
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestInsertDedupPolicy.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestInsertDedupPolicy.scala
index 99b68a3bbd42..995733dd45d1 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestInsertDedupPolicy.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestInsertDedupPolicy.scala
@@ -135,8 +135,6 @@ object TestInsertDedupPolicy {
def validate(expectedDf: Dataset[Row], actualDf: Dataset[Row]): Unit = {
val expectedMinusActual = expectedDf.except(actualDf)
val actualMinusExpected = actualDf.except(expectedDf)
- expectedDf.show(false)
- actualDf.show(false)
assertTrue(expectedMinusActual.isEmpty && actualMinusExpected.isEmpty)
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
index 81fd89ec6d41..188726989205 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
@@ -93,7 +93,6 @@ class PartitionStatsIndexTestBase extends
HoodieStatsIndexTestBase {
.option(OPERATION.key, operation)
.mode(saveMode)
.save(basePath)
- latestBatchDf.show(false)
val deletedDf = calculateMergedDf(latestBatchDf, operation)
deletedDf.cache()
if (validate) {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
index b434801ec056..bca39bd0221e 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
@@ -66,14 +66,6 @@ class TestAutoGenerationOfRecordKeys extends
HoodieSparkClientTestBase with Scal
initHoodieStorage()
}
- @AfterEach override def tearDown() = {
- cleanupSparkContexts()
- cleanupTestDataGenerator()
- cleanupFileSystem()
- FileSystem.closeAll()
- System.gc()
- }
-
@ParameterizedTest
@CsvSource(value = Array(
"AVRO,insert,COPY_ON_WRITE", "AVRO,bulk_insert,COPY_ON_WRITE",
"AVRO,insert,MERGE_ON_READ", "AVRO,bulk_insert,MERGE_ON_READ"
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
index 34963c7b4e22..328d969763f2 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
@@ -75,14 +75,6 @@ class TestBasicSchemaEvolution extends
HoodieSparkClientTestBase with ScalaAsser
initHoodieStorage()
}
- @AfterEach override def tearDown(): Unit = {
- cleanupSparkContexts()
- cleanupTestDataGenerator()
- cleanupFileSystem()
- FileSystem.closeAll()
- System.gc()
- }
-
// TODO add test-case for upcasting
@ParameterizedTest
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index a0485e8751b1..843532c4b625 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -1092,7 +1092,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase
with ScalaAssertionSup
val allRecords =
spark.read.format("org.apache.hudi").options(readOpts).load(basePath)
allRecords.registerTempTable("tmpTable")
- spark.sql(String.format("select count(*) from tmpTable")).show()
+ spark.sql(String.format("select count(*) from tmpTable")).collect()
// step4: Query the rows count from hoodie table for partition1
DEFAULT_FIRST_PARTITION_PATH
val recordCountForPartition1 = spark.sql(String.format("select count(*)
from tmpTable where partition = '%s'",
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).collect()
@@ -1141,7 +1141,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase
with ScalaAssertionSup
val allRecords =
spark.read.format("org.apache.hudi").options(readOpts).load(basePath)
allRecords.registerTempTable("tmpTable")
- spark.sql(String.format("select count(*) from tmpTable")).show()
+ spark.sql(String.format("select count(*) from tmpTable")).collect()
// step3: Query the rows count from hoodie table for partition1
DEFAULT_FIRST_PARTITION_PATH
val recordCountForPartition1 = spark.sql(String.format("select count(*)
from tmpTable where partition = '%s'",
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).collect()
@@ -1237,7 +1237,6 @@ class TestCOWDataSource extends HoodieSparkClientTestBase
with ScalaAssertionSup
.save(basePath)
val recordsReadDF =
spark.read.format("org.apache.hudi").options(readOpts).load(basePath)
- recordsReadDF.printSchema()
recordsReadDF.schema.foreach(f => {
f.name match {
case "timeStampValue" =>
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
index e1094782f252..00ee47e9fcb8 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
@@ -271,7 +271,7 @@ class TestMORDataSource extends HoodieSparkClientTestBase
with SparkDatasetMixin
assertEquals(100, hudiIncDF1.count())
assertEquals(1,
hudiIncDF1.select("_hoodie_commit_time").distinct().count())
assertEquals(commit1Time,
hudiIncDF1.select("_hoodie_commit_time").head().get(0).toString)
- hudiIncDF1.show(1)
+ hudiIncDF1.limit(1).collect
// log file only
val hudiIncDF2 = spark.read.format("org.apache.hudi")
@@ -283,7 +283,7 @@ class TestMORDataSource extends HoodieSparkClientTestBase
with SparkDatasetMixin
assertEquals(100, hudiIncDF2.count())
assertEquals(1,
hudiIncDF2.select("_hoodie_commit_time").distinct().count())
assertEquals(commit2Time,
hudiIncDF2.select("_hoodie_commit_time").head().get(0).toString)
- hudiIncDF2.show(1)
+ hudiIncDF2.limit(1).collect
// base file + log file
val hudiIncDF3 = spark.read.format("org.apache.hudi")
@@ -730,8 +730,8 @@ class TestMORDataSource extends HoodieSparkClientTestBase
with SparkDatasetMixin
assertEquals(sampleRow.getBoolean(2), sampleRow.get(2))
// test show()
- hudiSnapshotDF1.show(1)
- hudiSnapshotDF2.show(1)
+ hudiSnapshotDF1.limit(1).collect
+ hudiSnapshotDF2.limit(1).collect
}
@ParameterizedTest
@@ -810,8 +810,8 @@ class TestMORDataSource extends HoodieSparkClientTestBase
with SparkDatasetMixin
}
def verifyShow(df: DataFrame): Unit = {
- df.show(1)
- df.select("_hoodie_commit_seqno", "fare.amount", "fare.currency",
"tip_history").show(1)
+ df.limit(1).collect()
+ df.select("_hoodie_commit_seqno", "fare.amount", "fare.currency",
"tip_history").limit(1).collect()
}
@ParameterizedTest
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
index 9ed6ed282ef7..0e8f4b2c3021 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
@@ -60,14 +60,6 @@ class TestPartialUpdateAvroPayload extends
HoodieClientTestBase {
initHoodieStorage()
}
- @AfterEach override def tearDown() = {
- cleanupSparkContexts()
- cleanupTestDataGenerator()
- cleanupFileSystem()
- FileSystem.closeAll()
- System.gc()
- }
-
@ParameterizedTest
@EnumSource(value = classOf[HoodieTableType])
def testPartialUpdatesAvroPayloadPrecombine(hoodieTableType:
HoodieTableType): Unit = {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala
index f06b3e09b93b..0b2b163405bf 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala
@@ -80,15 +80,6 @@ class TestSparkDataSourceDAGExecution extends
HoodieSparkClientTestBase with Sca
initHoodieStorage()
}
- @AfterEach
- override def tearDown(): Unit = {
- cleanupSparkContexts()
- cleanupTestDataGenerator()
- cleanupFileSystem()
- FileSystem.closeAll()
- System.gc()
- }
-
@ParameterizedTest
@CsvSource(Array(
"upsert,org.apache.hudi.client.SparkRDDWriteClient.commit",
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
index 9e1ac11086cc..51905c00caa4 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
@@ -825,7 +825,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.option("hoodie.datasource.read.end.instanttime", startTimeStamp)
.option("hoodie.datasource.query.incremental.format", "cdc")
.load(basePath)
- result1.show(false)
+ result1.collect()
assertCDCOpCnt(result1, 1, 0, 0)
assertEquals(result1.count(), 1)
@@ -835,7 +835,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.option("hoodie.datasource.read.end.instanttime", latestTimeStamp)
.option("hoodie.datasource.query.incremental.format", "cdc")
.load(basePath)
- result2.show(false)
+ result2.collect()
assertCDCOpCnt(result2, 2, 1, 0)
assertEquals(result2.count(), 3)
@@ -845,7 +845,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.option("hoodie.datasource.read.end.instanttime", latestTimeStamp)
.option("hoodie.datasource.query.incremental.format", "cdc")
.load(basePath)
- result3.show(false)
+ result3.collect()
assertCDCOpCnt(result3, 3, 1, 0)
assertEquals(result3.count(), 4)
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
index 5e99dbe525d4..6e81e85aa152 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
@@ -166,7 +166,7 @@ class TestCDCStreamingSuite extends HoodieCDCTestBase with
SparkAdapterSupport {
assert(detailOutput1.where("country = 'US'").count() == 5)
val ucTs1 =
userToCountryMetaClient.reloadActiveTimeline().lastInstant.get.requestedTime
val ucDdcData1 = cdcDataFrame(userToCountryTblPath, (ucTs1.toLong -
1).toString, null)
- ucDdcData1.show(false)
+ ucDdcData1.collect()
assertCDCOpCnt(ucDdcData1, 1, 2, 0)
// check the final data of country_to_population_tbl for batch1
@@ -189,7 +189,7 @@ class TestCDCStreamingSuite extends HoodieCDCTestBase with
SparkAdapterSupport {
// check the change data about user_to_country_tbl for batch2
val ts2 =
userToCountryMetaClient.reloadActiveTimeline().lastInstant.get.requestedTime
val cdcData2 = cdcDataFrame(userToCountryTblPath, (ts2.toLong -
1).toString, null)
- cdcData2.show(false)
+ cdcData2.collect()
assertCDCOpCnt(cdcData2, 2, 1, 0)
// check the final data of country_to_population_tbl for batch2
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
index 45c4d4cd23f1..665a5e01385d 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
@@ -604,7 +604,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
| orderingFields = 'ts'
| )
""".stripMargin)
- spark.sql(s"show create table ${tableName}").show(false)
+ spark.sql(s"show create table ${tableName}").collect
spark.sql(s"insert into ${tableName} values (1, 'jack', 0.9, 1000)")
spark.sql(s"update ${tableName} set price = 1.9 where id = 1")
@@ -714,7 +714,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
| orderingFields = 'ts'
| )
""".stripMargin)
- spark.sql(s"show create table ${tableName}").show(false)
+ spark.sql(s"show create table ${tableName}").collect
spark.sql(s"insert into ${tableName} values (1, 'aaa', 'bbb', 1000)")
// Rename to a previously existing column name + insert
@@ -769,7 +769,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
// drop column
spark.sql(s"alter table ${tableName} drop columns(name, userx.name,
userx.score)")
- spark.sql(s"select * from ${tableName}").show(false)
+ spark.sql(s"select * from ${tableName}").collect()
// add cols back, and adjust cols position
spark.sql(s"alter table ${tableName} add columns(name string comment
'add name back' after userx," +
@@ -798,7 +798,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
spark.sql(s"insert into ${tableName} values(2, map('k1', struct(100,
'v1'), 'k2', struct(200, 'v2')), struct('jackStructNew', 291 , 101), 'jacknew',
1000, map('t1', 9))")
spark.sql(s"alter table ${tableName} alter column mxp.value type
double")
spark.sql(s"insert into ${tableName} values(2, map('k1', struct(100,
'v1'), 'k2', struct(200, 'v2')), struct('jackStructNew', 291 , 101), 'jacknew',
1000, map('t1', 10))")
- spark.sql(s"select * from $tableName").show(false)
+ spark.sql(s"select * from $tableName").collect
checkAnswer(spark.sql(s"select mxp from ${tableName} order by
id").collect())(
Seq(null),
Seq(Map("t1" -> 10.0d))
@@ -809,7 +809,6 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
spark.sql(s"alter table ${tableName} rename column us.age to age1")
spark.sql(s"insert into ${tableName} values(2, map('k1', struct(100,
'v1'), 'k2', struct(200, 'v2')), struct('jackStructNew', 291 , 101), 'jacknew',
1000, map('t1', 10))")
- spark.sql(s"select mem.value.nn, us.age1 from $tableName order by
id").show()
checkAnswer(spark.sql(s"select mem.value.nn, us.age1 from $tableName
order by id").collect())(
Seq(null, 29),
Seq(null, 291)
@@ -941,7 +940,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
Seq(checkRowKey, "new")
)
- spark.sql(s"select * from hudi_trips_snapshot").show(false)
+ spark.sql(s"select * from hudi_trips_snapshot").collect
// test insert_over_write + update again
val overwrite =
QuickstartUtils.convertToStringList(dataGen.generateInserts(10)).asScala.toSeq
val dfOverWrite = spark.
@@ -960,7 +959,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
option(DataSourceWriteOptions.TABLE_NAME.key(), tableName).
mode("append").
save(tablePath)
- spark.read.format("hudi").load(tablePath).show(false)
+ spark.read.format("hudi").load(tablePath).collect
val updatesAgain =
QuickstartUtils.convertToStringList(dataGen.generateUpdates(10)).asScala.toSeq
val dfAgain =
spark.read.json(spark.sparkContext.parallelize(updatesAgain, 2)).
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestCDCForSparkSQL.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestCDCForSparkSQL.scala
index a6b3e1654ff1..d082e7b175f5 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestCDCForSparkSQL.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/TestCDCForSparkSQL.scala
@@ -84,7 +84,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
val fgForID1 = spark.sql(s"select _hoodie_file_name from $tableName
where id=1").head().get(0)
val commitTime1 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly1 = cdcDataFrame(basePath, "000")
- cdcDataOnly1.show(false)
+ cdcDataOnly1.collect
assertCDCOpCnt(cdcDataOnly1, 2, 0, 0)
spark.sql(s"delete from $tableName where id = 1")
@@ -139,7 +139,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
spark.sql(s"insert into $tableName values (1, 'a1', 11, 1000), (2,
'a2', 12, 1000), (3, 'a3', 13, 1000)")
val commitTime1 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly1 = cdcDataFrame(basePath, "000")
- cdcDataOnly1.show(false)
+ cdcDataOnly1.collect
assertCDCOpCnt(cdcDataOnly1, 3, 0, 0)
spark.sql(s"insert into $tableName values (1, 'a1_v2', 11, 1100)")
@@ -147,7 +147,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
// here we use `commitTime1` to query the change data in commit 2.
// because `commitTime2` is maybe the ts of the compaction
operation, not the write operation.
val cdcDataOnly2 = cdcDataFrame(basePath, commitTime1)
- cdcDataOnly2.show(false)
+ cdcDataOnly2.collect
assertCDCOpCnt(cdcDataOnly2, 0, 1, 0)
// Check the details
@@ -169,13 +169,13 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
spark.sql(s"update $tableName set name = 'a2_v2', ts = 1200 where id
= 2")
val commitTime3 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly3 = cdcDataFrame(basePath, commitTime2)
- cdcDataOnly3.show(false)
+ cdcDataOnly3.collect
assertCDCOpCnt(cdcDataOnly3, 0, 1, 0)
spark.sql(s"delete from $tableName where id = 3")
val commitTime4 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly4 = cdcDataFrame(basePath, commitTime3)
- cdcDataOnly4.show(false)
+ cdcDataOnly4.collect
assertCDCOpCnt(cdcDataOnly4, 0, 0, 1)
spark.sql(
@@ -193,7 +193,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| when not matched then insert *
""".stripMargin)
val cdcDataOnly5 = cdcDataFrame(basePath, commitTime4)
- cdcDataOnly5.show(false)
+ cdcDataOnly5.collect
assertCDCOpCnt(cdcDataOnly5, 1, 1, 0)
// Check the details
@@ -266,19 +266,19 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
""".stripMargin)
val commitTime1 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly1 = cdcDataFrame(basePath, "000")
- cdcDataOnly1.show(false)
+ cdcDataOnly1.collect
assertCDCOpCnt(cdcDataOnly1, 3, 0, 0)
spark.sql(s"insert overwrite table $tableName partition (pt =
'2021') values (1, 'a1_v2', 11, 1100)")
val commitTime2 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly2 = cdcDataFrame(basePath, commitTime1)
- cdcDataOnly2.show(false)
+ cdcDataOnly2.collect
assertCDCOpCnt(cdcDataOnly2, 1, 0, 1)
spark.sql(s"update $tableName set name = 'a2_v2', ts = 1200 where id
= 2")
val commitTime3 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly3 = cdcDataFrame(basePath, commitTime2)
- cdcDataOnly3.show(false)
+ cdcDataOnly3.collect
assertCDCOpCnt(cdcDataOnly3, 0, 1, 0)
spark.sql(
@@ -296,7 +296,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| when not matched then insert *
""".stripMargin)
val cdcDataOnly4 = cdcDataFrame(basePath, commitTime3)
- cdcDataOnly4.show(false)
+ cdcDataOnly4.collect
assertCDCOpCnt(cdcDataOnly4, 1, 1, 0)
val totalCdcData = cdcDataFrame(basePath, "000")
@@ -349,19 +349,19 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
""".stripMargin)
val commitTime1 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly1 = cdcDataFrame(basePath, "000")
- cdcDataOnly1.show(false)
+ cdcDataOnly1.collect
assertCDCOpCnt(cdcDataOnly1, 3, 0, 0)
spark.sql(s"insert overwrite table $tableName partition (pt =
'2021') values (1, 'a1_v2', 11, 1100)")
val commitTime2 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly2 = cdcDataFrame(basePath, commitTime1)
- cdcDataOnly2.show(false)
+ cdcDataOnly2.collect
assertCDCOpCnt(cdcDataOnly2, 1, 0, 1)
spark.sql(s"update $tableName set name = 'a2_v2', ts = 1200 where id
= 2")
val commitTime3 =
metaClient.reloadActiveTimeline.lastInstant().get().requestedTime
val cdcDataOnly3 = cdcDataFrame(basePath, commitTime2)
- cdcDataOnly3.show(false)
+ cdcDataOnly3.collect
assertCDCOpCnt(cdcDataOnly3, 0, 1, 0)
spark.sql(
@@ -376,7 +376,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| when matched then update set id = s0.id, name = s0.name, ts =
s0.ts, pt = s0.pt
""".stripMargin)
val cdcDataOnly4 = cdcDataFrame(basePath, commitTime3)
- cdcDataOnly4.show(false)
+ cdcDataOnly4.collect
assertCDCOpCnt(cdcDataOnly4, 0, 1, 0)
val totalCdcData = cdcDataFrame(basePath, "000")
diff --git
a/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties
b/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties
index 287af214fb1d..c1129b2488f1 100644
--- a/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties
+++ b/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties
@@ -22,7 +22,7 @@ name = HudiConsoleLog
appender.console.type = Console
appender.console.name = CONSOLE
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n
+appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n%throwable{20}
# Root logger level
rootLogger.level = warn
diff --git a/hudi-tests-common/src/main/resources/log4j2-surefire.properties
b/hudi-tests-common/src/main/resources/log4j2-surefire.properties
index 6b6b2fa5e532..c391b354f2a1 100644
--- a/hudi-tests-common/src/main/resources/log4j2-surefire.properties
+++ b/hudi-tests-common/src/main/resources/log4j2-surefire.properties
@@ -22,7 +22,7 @@ name = HudiConsoleLog
appender.console.type = Console
appender.console.name = CONSOLE
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n
+appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n%throwable{20}
# Root logger level
rootLogger.level = warn
