This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2acaa752db2 [HUDI-6820] Fixing CI stability issues (#9661)
2acaa752db2 is described below
commit 2acaa752db2ff1664d82f12ea54d4a76fba05e21
Author: Lokesh Jain <[email protected]>
AuthorDate: Sat Sep 9 08:43:29 2023 +0530
[HUDI-6820] Fixing CI stability issues (#9661)
- We face frequent flakiness around 2 modules (hudi-hadoop-mr and
hudi-java-client). so, moving them out to github actions from azure CI.
- Added explicit timeouts for few of deltastreamer continuous tests so that
those fail instead of timing out.
---------
Co-authored-by: sivabalan <[email protected]>
---
.github/workflows/bot.yml | 32 ++++++++++++++++++++++
azure-pipelines-20230430.yml | 2 ++
.../deltastreamer/TestHoodieDeltaStreamer.java | 5 ++++
3 files changed, 39 insertions(+)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 0811c828e49..acd51b8e123 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -112,6 +112,38 @@ jobs:
run:
mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ test-hudi-hadoop-mr-and-hudi-java-client:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.2"
+ flinkProfile: "flink1.17"
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS
+ - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS
+
test-spark-java17:
runs-on: ubuntu-latest
strategy:
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 2da5ab0d4f9..25a149b5cf4 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -53,6 +53,8 @@ parameters:
- name: job4UTModules
type: object
default:
+ - '!hudi-hadoop-mr'
+ - '!hudi-client/hudi-java-client'
- '!hudi-client/hudi-spark-client'
- '!hudi-common'
- '!hudi-examples'
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 6324fb83fc9..2a7db25647e 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -120,6 +120,7 @@ import org.apache.spark.sql.types.StructField;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.CsvSource;
@@ -869,6 +870,7 @@ public class TestHoodieDeltaStreamer extends
HoodieDeltaStreamerTestBase {
defaultSchemaProviderClassName = FilebasedSchemaProvider.class.getName();
}
+ @Timeout(600)
@ParameterizedTest
@EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"})
public void testUpsertsCOWContinuousMode(HoodieRecordType recordType) throws
Exception {
@@ -892,12 +894,14 @@ public class TestHoodieDeltaStreamer extends
HoodieDeltaStreamerTestBase {
UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath);
}
+ @Timeout(600)
@ParameterizedTest
@EnumSource(value = HoodieRecordType.class, names = {"AVRO"})
public void testUpsertsMORContinuousModeShutdownGracefully(HoodieRecordType
recordType) throws Exception {
testUpsertsContinuousMode(HoodieTableType.MERGE_ON_READ, "continuous_cow",
true, recordType);
}
+ @Timeout(600)
@ParameterizedTest
@EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"})
public void testUpsertsMORContinuousMode(HoodieRecordType recordType) throws
Exception {
@@ -1404,6 +1408,7 @@ public class TestHoodieDeltaStreamer extends
HoodieDeltaStreamerTestBase {
UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath);
}
+ @Timeout(600)
@ParameterizedTest
@EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"})
public void testAsyncClusteringServiceWithCompaction(HoodieRecordType
recordType) throws Exception {
