This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new e604110af RATIS-2217. Automatically re-try flaky tests in CI (#1229)
e604110af is described below
commit e604110af2a31708e047858db45f4d47498495df
Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com>
AuthorDate: Wed Feb 26 18:40:39 2025 +0100
RATIS-2217. Automatically re-try flaky tests in CI (#1229)
---
.github/workflows/ci.yaml | 1 +
dev-support/checks/unit.sh | 4 +++
pom.xml | 18 ++++++++++
ratis-assembly/pom.xml | 2 ++
ratis-client/pom.xml | 5 +++
.../test/java/org/apache/ratis/test/tag/Flaky.java | 41 ++++++++++++++++++++++
.../java/org/apache/ratis/test/tag/FlakyTest.java | 28 +++++++++++++++
ratis-docs/pom.xml | 6 ++++
.../java/org/apache/ratis/TestMultiRaftGroup.java | 2 ++
ratis-experiments/pom.xml | 5 +++
ratis-metrics-api/pom.xml | 5 +++
ratis-netty/pom.xml | 5 +++
ratis-proto/pom.xml | 2 ++
ratis-replicated-map/pom.xml | 5 +++
ratis-server-api/pom.xml | 5 +++
.../ratis/server/impl/LeaderElectionTests.java | 2 ++
ratis-shell/pom.xml | 5 +++
.../org/apache/ratis/grpc/TestRaftWithGrpc.java | 2 ++
ratis-tools/pom.xml | 5 +++
19 files changed, 148 insertions(+)
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 55d55336b..cf230d384 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -83,6 +83,7 @@ jobs:
- grpc
- server
- misc
+ - flaky
fail-fast: false
uses: ./.github/workflows/check.yaml
with:
diff --git a/dev-support/checks/unit.sh b/dev-support/checks/unit.sh
index c652136da..c0369898e 100755
--- a/dev-support/checks/unit.sh
+++ b/dev-support/checks/unit.sh
@@ -36,6 +36,10 @@ mkdir -p "$REPORT_DIR"
export MAVEN_OPTS="-Xmx4096m"
MAVEN_OPTIONS='-V -B'
+if [[ "$@" =~ "-Pflaky-tests" ]]; then
+ MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dsurefire.rerunFailingTestsCount=5
-Dsurefire.timeout=1200"
+fi
+
if [[ "${FAIL_FAST}" == "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast
-Dsurefire.skipAfterFailureCount=1"
else
diff --git a/pom.xml b/pom.xml
index 0d5a0443e..386c7cf71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -222,6 +222,7 @@
<slf4j.version>2.0.7</slf4j.version>
<junit-bom.version>5.11.2</junit-bom.version>
<jacoco.version>0.8.12</jacoco.version>
+ <flaky-test-groups>flaky |
org.apache.ratis.test.tag.FlakyTest</flaky-test-groups>
</properties>
<dependencyManagement>
@@ -1118,6 +1119,7 @@
<includes>
<include>org.apache.ratis.grpc.**</include>
</includes>
+ <excludedGroups>${flaky-test-groups}</excludedGroups>
</configuration>
</plugin>
</plugins>
@@ -1135,6 +1137,7 @@
<include>org.apache.ratis.datastream.**</include>
<include>org.apache.ratis.server.**</include>
</includes>
+ <excludedGroups>${flaky-test-groups}</excludedGroups>
</configuration>
</plugin>
</plugins>
@@ -1153,6 +1156,21 @@
<exclude>org.apache.ratis.grpc.**</exclude>
<exclude>org.apache.ratis.server.**</exclude>
</excludes>
+ <excludedGroups>${flaky-test-groups}</excludedGroups>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>flaky-tests</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <groups>${flaky-test-groups}</groups>
</configuration>
</plugin>
</plugins>
diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml
index 1492f82b1..2b735fcb1 100644
--- a/ratis-assembly/pom.xml
+++ b/ratis-assembly/pom.xml
@@ -33,6 +33,8 @@
<test.cache.data>${project.build.directory}/test-classes</test.cache.data>
<test.build.classes>${project.build.directory}/test-classes</test.build.classes>
<license.bundles.dependencies>true</license.bundles.dependencies>
+ <!-- no testable code in this module -->
+ <skipTests>true</skipTests>
</properties>
<build>
diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml
index 3fc45790e..1456e682a 100644
--- a/ratis-client/pom.xml
+++ b/ratis-client/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-client</artifactId>
<name>Apache Ratis Client</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java
b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java
new file mode 100644
index 000000000..2d8c63030
--- /dev/null
+++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.test.tag;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Tag;
+
+/**
+ * Annotation to mark JUnit5 test classes or methods that exhibit intermittent
+ * issues. These are run separately from the normal tests in CI. In case of
+ * failure they may be repeated a few times.
+ * Usage: <code>@Flaky("RATIS-123")</code>
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("flaky")
+public @interface Flaky {
+ /**
+ * The issue(s) tracking the flaky test.
+ */
+ String[] value();
+}
diff --git
a/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java
b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java
new file mode 100644
index 000000000..f43bd1698
--- /dev/null
+++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.test.tag;
+
+/**
+ * Interface to mark JUnit4 test classes or methods that exhibit intermittent
+ * issues. These are run separately from the normal tests in CI. In case of
+ * failure they may be repeated a few times.
+ * Usage: <code>@Category(FlakyTest.class) @Flaky("RATIS-123")</code>
+ */
+public interface FlakyTest {
+ // category marker
+}
diff --git a/ratis-docs/pom.xml b/ratis-docs/pom.xml
index d9fc447ae..1bb1c6458 100644
--- a/ratis-docs/pom.xml
+++ b/ratis-docs/pom.xml
@@ -26,4 +26,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>ratis-docs</artifactId>
<name>Apache Ratis Documentation</name>
<packaging>jar</packaging>
+
+ <properties>
+ <!-- no testable code in this module -->
+ <skipTests>true</skipTests>
+ </properties>
+
</project>
diff --git
a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java
b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java
index ea3962c08..5ce96da39 100644
--- a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java
+++ b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java
@@ -24,6 +24,7 @@ import org.apache.ratis.examples.arithmetic.TestArithmetic;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.server.impl.GroupManagementBaseTest;
import org.apache.ratis.server.impl.MiniRaftCluster;
+import org.apache.ratis.test.tag.Flaky;
import org.apache.ratis.util.function.CheckedBiConsumer;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
@@ -33,6 +34,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
+@Flaky("RATIS-2218")
@Timeout(value = 300)
public class TestMultiRaftGroup extends BaseTest {
public static Collection<Object[]> data() {
diff --git a/ratis-experiments/pom.xml b/ratis-experiments/pom.xml
index 086cff583..39ad9668f 100644
--- a/ratis-experiments/pom.xml
+++ b/ratis-experiments/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-experiments</artifactId>
<name>Apache Ratis Experiments</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
diff --git a/ratis-metrics-api/pom.xml b/ratis-metrics-api/pom.xml
index 19cecf984..cc11e91d5 100644
--- a/ratis-metrics-api/pom.xml
+++ b/ratis-metrics-api/pom.xml
@@ -24,6 +24,11 @@
<artifactId>ratis-metrics-api</artifactId>
<name>Apache Ratis Metrics API</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<artifactId>ratis-common</artifactId>
diff --git a/ratis-netty/pom.xml b/ratis-netty/pom.xml
index 1a29eb8eb..482fcb468 100644
--- a/ratis-netty/pom.xml
+++ b/ratis-netty/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-netty</artifactId>
<name>Apache Ratis Netty Support</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
diff --git a/ratis-proto/pom.xml b/ratis-proto/pom.xml
index e3730fae0..e3342f84b 100644
--- a/ratis-proto/pom.xml
+++ b/ratis-proto/pom.xml
@@ -25,6 +25,8 @@
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
+ <!-- no testable code in this module -->
+ <skipTests>true</skipTests>
<spotbugs.skip>true</spotbugs.skip>
</properties>
diff --git a/ratis-replicated-map/pom.xml b/ratis-replicated-map/pom.xml
index bded7cd36..e6144fd10 100644
--- a/ratis-replicated-map/pom.xml
+++ b/ratis-replicated-map/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-replicated-map</artifactId>
<name>Apache Ratis Replicated Map</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
diff --git a/ratis-server-api/pom.xml b/ratis-server-api/pom.xml
index 801efe81f..e486d0322 100644
--- a/ratis-server-api/pom.xml
+++ b/ratis-server-api/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-server-api</artifactId>
<name>Apache Ratis Server API</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
diff --git
a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java
b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java
index bda496c16..c51ffda64 100644
---
a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java
+++
b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java
@@ -37,6 +37,7 @@ import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.metrics.LeaderElectionMetrics;
import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.test.tag.Flaky;
import org.apache.ratis.thirdparty.com.codahale.metrics.Timer;
import org.apache.ratis.util.CodeInjectionForTesting;
import org.apache.ratis.util.JavaUtils;
@@ -729,6 +730,7 @@ public abstract class LeaderElectionTests<CLUSTER extends
MiniRaftCluster>
}
}
+ @Flaky("RATIS-2108")
@Test
public void testLeaderLeaseDuringReconfiguration() throws Exception {
// use a strict lease
diff --git a/ratis-shell/pom.xml b/ratis-shell/pom.xml
index de92e5b0c..41c2a0ec3 100644
--- a/ratis-shell/pom.xml
+++ b/ratis-shell/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-shell</artifactId>
<name>Apache Ratis Shell</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<artifactId>ratis-client</artifactId>
diff --git
a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
index 5821b7f22..c13544a78 100644
--- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
+++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
@@ -28,6 +28,7 @@ import org.apache.ratis.server.raftlog.LogEntryHeader;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
import org.apache.ratis.statemachine.StateMachine;
+import org.apache.ratis.test.tag.Flaky;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;
import org.junit.jupiter.api.Assertions;
@@ -55,6 +56,7 @@ public class TestRaftWithGrpc
// skip testWithLoad() from parent, called from parameterized
testWithLoad(boolean)
}
+ @Flaky("RATIS-2253")
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testWithLoad(boolean separateHeartbeat) throws Exception {
diff --git a/ratis-tools/pom.xml b/ratis-tools/pom.xml
index 8f39cac68..f3dbb3188 100644
--- a/ratis-tools/pom.xml
+++ b/ratis-tools/pom.xml
@@ -23,6 +23,11 @@
<artifactId>ratis-tools</artifactId>
<name>Apache Ratis Tools</name>
+ <properties>
+ <!-- no tests in this module so far -->
+ <skipTests>true</skipTests>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ratis</groupId>
