This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7f246d5419 Harry Simulation test halts the JVM when errors are
detected which loose all history in CI
7f246d5419 is described below
commit 7f246d541985eb43d6867338f9900d3c93ba10d9
Author: ci worker <[email protected]>
AuthorDate: Thu Jul 25 10:28:35 2024 -0700
Harry Simulation test halts the JVM when errors are detected which loose
all history in CI
patch by David Capwell; reviewed by Alex Petrov for CASSANDRA-19802
---
build.xml | 2 +
.../cassandra/simulator/SimulationRunner.java | 15 +++--
.../simulator/harry/HarryValidatingQuery.java | 8 +--
.../simulator/test/HarrySimulatorTest.java | 70 +++++++++++++---------
.../simulator/test/SimulationTestBase.java | 2 +-
5 files changed, 59 insertions(+), 38 deletions(-)
diff --git a/build.xml b/build.xml
index 043188d61c..13e0a9ebd8 100644
--- a/build.xml
+++ b/build.xml
@@ -1797,6 +1797,8 @@
<jvmarg line="-XX:Tier4CompileThreshold=1000"/>
<jvmarg line="-XX:ReservedCodeCacheSize=256M"/>
<jvmarg line="-Xmx8G"/>
+ <!-- Harry tests kept failing due to direct memory failures and looks
like its undersized... so upping to allow more stable runs -->
+ <jvmarg line="-XX:MaxDirectMemorySize=8G"/>
</testmacro>
</target>
diff --git
a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
index 026078c9b8..5542d0d1e2 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
@@ -433,13 +433,16 @@ public class SimulationRunner
}
- private static Optional<Long> parseHex(Optional<String> value)
+ public static Optional<Long> parseHex(Optional<String> value)
{
- return value.map(s -> {
- if (s.startsWith("0x"))
- return Hex.parseLong(s, 2, s.length());
- throw new IllegalArgumentException("Invalid hex string: " + s);
- });
+ return value.map(SimulationRunner::parseHex);
+ }
+
+ public static long parseHex(String s)
+ {
+ if (s.startsWith("0x"))
+ return Hex.parseLong(s, 2, s.length());
+ throw new IllegalArgumentException("Invalid hex string: " + s);
}
private static final Pattern CHANCE_PATTERN =
Pattern.compile("(uniform|(?<qlog>qlog(\\((?<quantizations>[0-9]+)\\))?):)?(?<min>0(\\.[0-9]+)?)(..(?<max>0\\.[0-9]+))?",
Pattern.CASE_INSENSITIVE);
diff --git
a/test/simulator/main/org/apache/cassandra/simulator/harry/HarryValidatingQuery.java
b/test/simulator/main/org/apache/cassandra/simulator/harry/HarryValidatingQuery.java
index 1ac7b550f6..05620b5517 100644
---
a/test/simulator/main/org/apache/cassandra/simulator/harry/HarryValidatingQuery.java
+++
b/test/simulator/main/org/apache/cassandra/simulator/harry/HarryValidatingQuery.java
@@ -113,8 +113,9 @@ public class HarryValidatingQuery extends SimulatedAction
if (!throwables.isEmpty())
{
- logger.error(String.format("Could not validate
%d out of %d replicas %s", throwables.size(), replicas.size(), replicas),
throwables.get(0));
- System.exit(0);
+ AssertionError error = new
AssertionError(String.format("Could not validate %d out of %d replicas %s",
throwables.size(), replicas.size(), replicas));
+ throwables.forEach(error::addSuppressed);
+ throw error;
}
}
@@ -137,9 +138,8 @@ public class HarryValidatingQuery extends SimulatedAction
catch (Throwable t)
{
logger.error("Caught an exception while validating", t);
- t.printStackTrace();
+ throw t;
}
- finally { }
}
};
}
diff --git
a/test/simulator/test/org/apache/cassandra/simulator/test/HarrySimulatorTest.java
b/test/simulator/test/org/apache/cassandra/simulator/test/HarrySimulatorTest.java
index 81e70079aa..358faec3de 100644
---
a/test/simulator/test/org/apache/cassandra/simulator/test/HarrySimulatorTest.java
+++
b/test/simulator/test/org/apache/cassandra/simulator/test/HarrySimulatorTest.java
@@ -27,6 +27,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -80,6 +82,7 @@ import org.apache.cassandra.simulator.OrderOn;
import org.apache.cassandra.simulator.RandomSource;
import org.apache.cassandra.simulator.RunnableActionScheduler;
import org.apache.cassandra.simulator.Simulation;
+import org.apache.cassandra.simulator.SimulationException;
import org.apache.cassandra.simulator.SimulationRunner;
import org.apache.cassandra.simulator.SimulatorUtils;
import
org.apache.cassandra.simulator.cluster.ClusterActionListener.NoOpListener;
@@ -187,6 +190,8 @@ public class HarrySimulatorTest
public int rowsPerPhase = 10;
@Option(name = {"--nodes-per-dc"}, description = "How many nodes per dc
for replication")
public int nodesPerDc = 3;
+ @Option(name = {"-s", "--seed"}, title = "0x", description = "What seed to
run with; in hex format... example: 0x190e6ff01d6")
+ public String seed = null;
public static void main(String... args) throws Throwable
{
@@ -201,6 +206,8 @@ public class HarrySimulatorTest
public void test() throws Exception
{
rowsPerPhase = 1;
+ // To rerun a failing test for a given seed, uncomment the below and
set the seed
+// this.seed = "<your seed here>";
harryTest();
}
@@ -436,7 +443,23 @@ public class HarrySimulatorTest
try (CloseableIterator<?> iter = iterator())
{
while (iter.hasNext())
+ {
+ checkForErrors();
iter.next();
+ }
+ checkForErrors();
+ }
+ }
+
+ private void checkForErrors()
+ {
+ if (simulated.failures.hasFailure())
+ {
+ AssertionError error = new AssertionError("Errors detected
during simulation");
+ // don't care about the stack trace... the issue is the errors
found and not what part of the scheduler we stopped
+ error.setStackTrace(new StackTraceElement[0]);
+ simulated.failures.get().forEach(error::addSuppressed);
+ throw error;
}
}
@@ -504,43 +527,38 @@ public class HarrySimulatorTest
/**
* Simulation entrypoint; syntax sugar for creating a simulation.
*/
- static void simulate(Consumer<ClusterSimulation.Builder<HarrySimulation>>
configure,
- Consumer<IInstanceConfig> instanceConfigUpdater,
- Configuration.ConfigurationBuilder harryConfig,
- String[] properties,
- Function<HarrySimulation, ActionSchedule.Work[]>...
phases) throws IOException
+ void simulate(Consumer<ClusterSimulation.Builder<HarrySimulation>>
configure,
+ Consumer<IInstanceConfig> instanceConfigUpdater,
+ Configuration.ConfigurationBuilder harryConfig,
+ String[] properties,
+ Function<HarrySimulation, ActionSchedule.Work[]>... phases)
throws IOException
{
try (WithProperties p = new WithProperties().with(properties))
{
HarrySimulationBuilder factory = new
HarrySimulationBuilder(harryConfig, instanceConfigUpdater);
SimulationRunner.beforeAll();
- long seed = System.currentTimeMillis();
- // Development seed:
- //long seed = 1687184561194L;
- System.out.println("Simulation seed: " + seed + "L");
+ long seed =
SimulationRunner.parseHex(Optional.ofNullable(this.seed)).orElseGet(() -> new
Random().nextLong());
+ logger.info("Seed 0x{}", Long.toHexString(seed));
configure.accept(factory);
try (ClusterSimulation<HarrySimulation> clusterSimulation =
factory.create(seed))
{
- try
- {
- HarrySimulation simulation =
clusterSimulation.simulation();
-
- // For better determinism during startup, we allow
instances to fully start (including daemon work)
- for (int i = 0; i < phases.length; i++)
- {
- HarrySimulation current = simulation;
- if (i == 0)
- current = current.withScheduler(new
RunnableActionScheduler.Immediate()).withSchedulers((s) ->
Collections.emptyMap());
- current.withSchedule(phases[i]).run();
- }
- }
- catch (Throwable t)
+ HarrySimulation simulation = clusterSimulation.simulation();
+
+ // For better determinism during startup, we allow instances
to fully start (including daemon work)
+ for (int i = 0; i < phases.length; i++)
{
- throw new AssertionError(String.format("Failed on seed
%s", Long.toHexString(seed)),
- t);
+ HarrySimulation current = simulation;
+ if (i == 0)
+ current = current.withScheduler(new
RunnableActionScheduler.Immediate()).withSchedulers((s) ->
Collections.emptyMap());
+ current.withSchedule(phases[i]).run();
}
}
+ catch (Throwable t)
+ {
+ if (t instanceof SimulationException) throw t;
+ throw new SimulationException(seed, "Failure creating the
simulation", t);
+ }
}
}
@@ -976,9 +994,7 @@ public class HarrySimulatorTest
public void onFailure(Throwable t)
{
super.onFailure(t);
- t.printStackTrace();
logger.error("Caught an exception, going to halt", t);
- //while (true) { }
}
}
}
diff --git
a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java
b/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java
index d1ebbf89cd..88aaf5374c 100644
---
a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java
+++
b/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java
@@ -206,7 +206,7 @@ public class SimulationTestBase
long seed = System.currentTimeMillis();
// Development seed:
//long seed = 1687184561194L;
- System.out.println("Simulation seed: " + seed + "L");
+ logger.info("Simulation seed: {}L", seed);
configure.accept(factory);
try (ClusterSimulation<?> cluster = factory.create(seed))
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
