This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit d4ecaf80e69ea3bd3d8a9538ae3f70851997b6cf
Author: Abe Ratnofsky <a...@aber.io>
AuthorDate: Mon Oct 17 12:54:44 2022 +0200
Improvements:
* formatting
* thread shutdown on failed run creation
* idempotent query execution
* retry delay
Patch by Abe Ratnofsky; reviewed by Alex Petrov for CASSANDRA-18315.
---
harry-core/src/harry/core/Configuration.java | 56 ++++++++++++++--------
harry-core/src/harry/ddl/ColumnSpec.java | 16 +++++++
harry-core/src/harry/ddl/SchemaGenerators.java | 41 ++++++++++------
harry-core/src/harry/generators/Bijections.java | 35 +++++++++++++-
harry-core/src/harry/generators/PCGFastPure.java | 2 +-
harry-core/src/harry/model/SelectHelper.java | 2 +-
.../src/harry/model/sut/SystemUnderTest.java | 12 ++++-
harry-core/src/harry/runner/HarryRunner.java | 6 +++
harry-core/src/harry/runner/Runner.java | 9 +++-
harry-core/src/harry/visitors/MutatingVisitor.java | 6 ++-
.../harry/visitors/ParallelRecentValidator.java | 4 +-
harry-core/src/harry/visitors/RecentValidator.java | 2 +-
12 files changed, 145 insertions(+), 46 deletions(-)
diff --git a/harry-core/src/harry/core/Configuration.java
b/harry-core/src/harry/core/Configuration.java
index 3545e18..7cd8f72 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -65,6 +65,8 @@ public class Configuration
{
private static final ObjectMapper mapper;
+ private static final String DEFAULT_KEYSPACE = "harry";
+
static
{
mapper = new ObjectMapper(new YAMLFactory()
@@ -220,34 +222,46 @@ public class Configuration
public static Run createRun(Configuration snapshot)
{
- validate(snapshot);
+ SystemUnderTest sut = null;
+ try
+ {
+ validate(snapshot);
- long seed = snapshot.seed;
+ long seed = snapshot.seed;
- DataTracker tracker = snapshot.data_tracker == null ? new
DefaultDataTrackerConfiguration().make() : snapshot.data_tracker.make();
- OpSelectors.Rng rng = new OpSelectors.PCGFast(seed);
+ DataTracker tracker = snapshot.data_tracker == null ? new
DefaultDataTrackerConfiguration().make() : snapshot.data_tracker.make();
+ OpSelectors.Rng rng = new OpSelectors.PCGFast(seed);
- OpSelectors.MonotonicClock clock = snapshot.clock.make();
+ OpSelectors.MonotonicClock clock = snapshot.clock.make();
- MetricReporter metricReporter = snapshot.metric_reporter.make();
+ MetricReporter metricReporter = snapshot.metric_reporter.make();
- // TODO: validate that operation kind is compatible with schema, due
to statics etc
- SystemUnderTest sut = snapshot.system_under_test.make();
+ // TODO: validate that operation kind is compatible with schema,
due to statics etc
+ sut = snapshot.system_under_test.make();
- SchemaSpec schemaSpec = snapshot.schema_provider.make(seed, sut);
- schemaSpec.validate();
+ SchemaSpec schemaSpec = snapshot.schema_provider.make(seed, sut);
+ schemaSpec.validate();
- OpSelectors.PdSelector pdSelector =
snapshot.partition_descriptor_selector.make(rng);
- OpSelectors.DescriptorSelector descriptorSelector =
snapshot.clustering_descriptor_selector.make(rng, schemaSpec);
+ OpSelectors.PdSelector pdSelector =
snapshot.partition_descriptor_selector.make(rng);
+ OpSelectors.DescriptorSelector descriptorSelector =
snapshot.clustering_descriptor_selector.make(rng, schemaSpec);
- return new Run(rng,
- clock,
- pdSelector,
- descriptorSelector,
- schemaSpec,
- tracker,
- sut,
- metricReporter);
+ return new Run(rng,
+ clock,
+ pdSelector,
+ descriptorSelector,
+ schemaSpec,
+ tracker,
+ sut,
+ metricReporter);
+ }
+ catch (Throwable t)
+ {
+ // Make sure to shut down all SUT threads if it has been started
+ if (sut != null) {
+ sut.shutdown();
+ }
+ throw t;
+ }
}
public static Runner createRunner(Configuration config)
@@ -1058,7 +1072,7 @@ public class Configuration
{
public SchemaSpec make(long seed, SystemUnderTest sut)
{
- return SchemaGenerators.defaultSchemaSpecGen("harry", "table0")
+ return SchemaGenerators.defaultSchemaSpecGen("table0")
.inflate(seed);
}
}
diff --git a/harry-core/src/harry/ddl/ColumnSpec.java
b/harry-core/src/harry/ddl/ColumnSpec.java
index 9b52805..7a2001c 100644
--- a/harry-core/src/harry/ddl/ColumnSpec.java
+++ b/harry-core/src/harry/ddl/ColumnSpec.java
@@ -318,6 +318,19 @@ public class ColumnSpec<T>
}
};
+ public static final DataType<UUID> timeUuidType = new
DataType<UUID>("timeuuid")
+ {
+ public Bijections.Bijection<UUID> generator()
+ {
+ return Bijections.TIME_UUID_GENERATOR;
+ }
+
+ public int compareLexicographically(long l, long r)
+ {
+ throw new RuntimeException("UUID does not support custom
comparators");
+ }
+ };
+
public static final DataType<Date> timestampType = new
DataType<Date>("timestamp")
{
public Bijections.Bijection<Date> generator()
@@ -342,6 +355,7 @@ public class ColumnSpec<T>
ColumnSpec.asciiType,
ColumnSpec.textType,
ColumnSpec.uuidType,
+ ColumnSpec.timeUuidType,
ColumnSpec.timestampType));
public static class ReversedType<T> extends DataType<T>
@@ -356,6 +370,8 @@ public class ColumnSpec<T>
put(floatType, new ReversedType<>(floatType, new
Bijections.ReverseFloatGenerator()));
put(doubleType, new ReversedType<>(doubleType, new
Bijections.ReverseDoubleGenerator()));
put(asciiType, new ReversedType<>(asciiType));
+ put(uuidType, new ReversedType<>(uuidType));
+ put(timeUuidType, new ReversedType<>(timeUuidType));
}};
private final DataType<T> baseType;
diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java
b/harry-core/src/harry/ddl/SchemaGenerators.java
index 8508d44..f56c0d5 100644
--- a/harry-core/src/harry/ddl/SchemaGenerators.java
+++ b/harry-core/src/harry/ddl/SchemaGenerators.java
@@ -48,25 +48,30 @@ public class SchemaGenerators
static
{
- partitionKeyTypes =
Collections.unmodifiableList(Arrays.asList(ColumnSpec.int64Type,
+ partitionKeyTypes =
Collections.unmodifiableList(Arrays.asList(ColumnSpec.int8Type,
+
ColumnSpec.int16Type,
+
ColumnSpec.int32Type,
+
ColumnSpec.int64Type,
+
ColumnSpec.floatType,
+
ColumnSpec.doubleType,
ColumnSpec.asciiType,
-
ColumnSpec.asciiType(4, 5),
-
ColumnSpec.asciiType(4, 10)));
+
ColumnSpec.textType));
- columnTypes = Collections.unmodifiableList(Arrays.asList(
-// ColumnSpec.int8Type,
-//
ColumnSpec.int16Type,
-//
ColumnSpec.int32Type,
+ columnTypes =
Collections.unmodifiableList(Arrays.asList(ColumnSpec.int8Type,
+
ColumnSpec.int16Type,
+
ColumnSpec.int32Type,
ColumnSpec.int64Type,
+
ColumnSpec.booleanType,
+
ColumnSpec.floatType,
+
ColumnSpec.doubleType,
ColumnSpec.asciiType,
-
ColumnSpec.asciiType(4, 256),
-
ColumnSpec.asciiType(4, 512)));
+
ColumnSpec.textType));
- List<ColumnSpec.DataType<?>> builder = new ArrayList<>(columnTypes);
+ List<ColumnSpec.DataType<?>> builder = new
ArrayList<>(partitionKeyTypes);
Map<String, ColumnSpec.DataType<?>> mapBuilder = new HashMap<>();
- for (ColumnSpec.DataType<?> columnType : columnTypes)
+ for (ColumnSpec.DataType<?> columnType : partitionKeyTypes)
{
ColumnSpec.DataType<?> reversedType =
ColumnSpec.ReversedType.getInstance(columnType);
builder.add(reversedType);
@@ -345,9 +350,9 @@ public class SchemaGenerators
}
}
- public static Surjections.Surjection<SchemaSpec>
defaultSchemaSpecGen(String ks, String table)
+ public static Surjections.Surjection<SchemaSpec>
defaultSchemaSpecGen(String table)
{
- return new SchemaGenerators.Builder(ks, () -> table)
+ return new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, () -> table)
.partitionKeySpec(1, 3,
columnTypes)
.clusteringKeySpec(1, 3,
@@ -372,7 +377,13 @@ public class SchemaGenerators
.surjection();
}
- public static final String DEFAULT_KEYSPACE_NAME = "harry";
+ public static String DEFAULT_KEYSPACE_NAME =
System.getProperty("harry.keyspace");
+ static {
+ if (DEFAULT_KEYSPACE_NAME == null || "".equals(DEFAULT_KEYSPACE_NAME))
+ DEFAULT_KEYSPACE_NAME = "harry";
+ }
+
+
private static final String DEFAULT_PREFIX = "table_";
private static final AtomicInteger counter = new AtomicInteger();
private static final Supplier<String> tableNameSupplier = () ->
DEFAULT_PREFIX + counter.getAndIncrement();
@@ -503,4 +514,4 @@ public class SchemaGenerators
public static int DEFAULT_SWITCH_AFTER =
Integer.getInteger("harry.test.progression.switch-after", 5);
public static int GENERATORS_COUNT = PROGRESSIVE_GENERATORS.length;
public static int DEFAULT_RUNS = DEFAULT_SWITCH_AFTER * GENERATORS_COUNT;
-}
\ No newline at end of file
+}
diff --git a/harry-core/src/harry/generators/Bijections.java
b/harry-core/src/harry/generators/Bijections.java
index 4cc06a7..572a664 100644
--- a/harry-core/src/harry/generators/Bijections.java
+++ b/harry-core/src/harry/generators/Bijections.java
@@ -32,6 +32,7 @@ public class Bijections
public static final Bijection<Boolean> BOOLEAN_GENERATOR = new
BooleanGenerator();
public static final Bijection<UUID> UUID_GENERATOR = new UUIDGenerator();
+ public static final TimeUUIDGenerator TIME_UUID_GENERATOR = new
TimeUUIDGenerator();
public static final Bijection<Date> TIMESTAMP_GENERATOR = new
TimestampGenerator();
/**
@@ -390,13 +391,45 @@ public class Bijections
public int compare(long l, long r)
{
- return Byte.compare((byte) l, (byte) r);
+ return Long.compare(l, r);
+ }
+
+ public int byteSize()
+ {
+ return Long.BYTES;
+ }
+ }
+
+ public static class TimeUUIDGenerator implements Bijection<UUID>
+ {
+ public UUID inflate(long current)
+ {
+ return new UUID(createTime(current), current);
+ }
+
+ public long deflate(UUID value)
+ {
+ return value.getLeastSignificantBits();
+ }
+
+ public int compare(long left, long right)
+ {
+ return Long.compare(left, right);
}
public int byteSize()
{
return Long.BYTES;
}
+
+ public static long createTime(long nanosSince)
+ {
+ long msb = nanosSince;
+ msb &= 0xffffffffffff10ffL; // sets the version to 1.
+ msb |= 0x0000000000001000L;
+ return msb;
+ }
+
}
public static class TimestampGenerator implements Bijection<Date>
diff --git a/harry-core/src/harry/generators/PCGFastPure.java
b/harry-core/src/harry/generators/PCGFastPure.java
index caed276..9e438dc 100644
--- a/harry-core/src/harry/generators/PCGFastPure.java
+++ b/harry-core/src/harry/generators/PCGFastPure.java
@@ -25,7 +25,7 @@ package harry.generators;
* https://github.com/imneme/pcg-c
* https://github.com/imneme/pcg-cpp
* <p>
- * Original library developed by Melissa O'Neill (one...@pcg-random.org)
+ * Original library developed by Melissa O'Neill <one...@pcg-random.org>
*/
public class PCGFastPure
{
diff --git a/harry-core/src/harry/model/SelectHelper.java
b/harry-core/src/harry/model/SelectHelper.java
index c0813ba..675052c 100644
--- a/harry-core/src/harry/model/SelectHelper.java
+++ b/harry-core/src/harry/model/SelectHelper.java
@@ -194,7 +194,7 @@ public class SelectHelper
public static List<ResultSetRow> execute(SystemUnderTest sut,
OpSelectors.MonotonicClock clock, Query query)
{
CompiledStatement compiled = query.toSelectStatement();
- Object[][] objects = sut.execute(compiled.cql(),
SystemUnderTest.ConsistencyLevel.QUORUM, compiled.bindings());
+ Object[][] objects = sut.executeIdempotent(compiled.cql(),
SystemUnderTest.ConsistencyLevel.QUORUM, compiled.bindings());
List<ResultSetRow> result = new ArrayList<>();
for (Object[] obj : objects)
result.add(resultSetToRow(query.schemaSpec, clock, obj));
diff --git a/harry-core/src/harry/model/sut/SystemUnderTest.java
b/harry-core/src/harry/model/sut/SystemUnderTest.java
index ea2e633..3bd6bf1 100644
--- a/harry-core/src/harry/model/sut/SystemUnderTest.java
+++ b/harry-core/src/harry/model/sut/SystemUnderTest.java
@@ -42,15 +42,25 @@ public interface SystemUnderTest
execute(statement, ConsistencyLevel.ALL, new Object[]{});
}
+ Object[][] execute(String statement, ConsistencyLevel cl, Object...
bindings);
+
default Object[][] execute(CompiledStatement statement, ConsistencyLevel
cl)
{
return execute(statement.cql(), cl, statement.bindings());
}
- Object[][] execute(String statement, ConsistencyLevel cl, Object...
bindings);
+ default Object[][] executeIdempotent(String statement, ConsistencyLevel
cl, Object... bindings)
+ {
+ return execute(statement, cl, bindings);
+ }
CompletableFuture<Object[][]> executeAsync(String statement,
ConsistencyLevel cl, Object... bindings);
+ default CompletableFuture<Object[][]> executeIdempotentAsync(String
statement, ConsistencyLevel cl, Object... bindings)
+ {
+ return executeAsync(statement, cl, bindings);
+ }
+
interface SystemUnderTestFactory
{
SystemUnderTest create();
diff --git a/harry-core/src/harry/runner/HarryRunner.java
b/harry-core/src/harry/runner/HarryRunner.java
index 30f7f86..4dbfe76 100644
--- a/harry-core/src/harry/runner/HarryRunner.java
+++ b/harry-core/src/harry/runner/HarryRunner.java
@@ -35,11 +35,17 @@ public abstract class HarryRunner
{
public static final Logger logger =
LoggerFactory.getLogger(HarryRunner.class);
+ protected final boolean localRun =
Boolean.parseBoolean(System.getProperty("harry.local-run", "true"));
+ protected final String cassandraVersion =
System.getProperty("harry.cassandra-version");
+ protected final long startedAt = Long.getLong("harry.start-time",
System.currentTimeMillis());
+
protected CompletableFuture<?> progress;
protected ScheduledThreadPoolExecutor executor;
public abstract void beforeRun(Runner.TimedRunner runner);
public void afterRun(Runner runner, Object result)
{
+ boolean success = !(result instanceof Throwable);
+
executor.shutdown();
try
{
diff --git a/harry-core/src/harry/runner/Runner.java
b/harry-core/src/harry/runner/Runner.java
index 4f99eab..207394d 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -69,6 +69,11 @@ public abstract class Runner
return run;
}
+ public int getErrorCount()
+ {
+ return errors.size();
+ }
+
public void init()
{
if (config.create_schema)
@@ -114,8 +119,9 @@ public abstract class Runner
protected void maybeReportErrors()
{
- if (!errors.isEmpty())
+ if (!errors.isEmpty()) {
dumpStateToFile(run, config, errors);
+ }
}
public CompletableFuture<?> initAndStartAll()
@@ -327,6 +333,7 @@ public abstract class Runner
public void shutdown() throws InterruptedException
{
logger.info("Shutting down...");
+
shutDownVisitors();
// we need to wait for all threads that use schema to stop before
we can tear down and drop the table
diff --git a/harry-core/src/harry/visitors/MutatingVisitor.java
b/harry-core/src/harry/visitors/MutatingVisitor.java
index d6eb617..db27216 100644
--- a/harry-core/src/harry/visitors/MutatingVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingVisitor.java
@@ -87,7 +87,7 @@ public class MutatingVisitor extends GeneratingVisitor
{
try
{
- future.get();
+ future.get(10, TimeUnit.SECONDS);
}
catch (Throwable t)
{
@@ -162,7 +162,9 @@ public class MutatingVisitor extends GeneratingVisitor
if (t != null)
{
logger.error("Caught message while trying to execute "
+ statement, t);
- executor.schedule(() -> executeAsyncWithRetries(future,
statement, retries + 1), 1, TimeUnit.SECONDS);
+ int delaySecs = 1;
+ executor.schedule(() -> executeAsyncWithRetries(future,
statement, retries + 1), delaySecs, TimeUnit.SECONDS);
+ logger.info("Scheduled retry to happen with delay {}
seconds", delaySecs);
}else
future.complete(res);
});
diff --git a/harry-core/src/harry/visitors/ParallelRecentValidator.java
b/harry-core/src/harry/visitors/ParallelRecentValidator.java
index 29503d1..a761c1e 100644
--- a/harry-core/src/harry/visitors/ParallelRecentValidator.java
+++ b/harry-core/src/harry/visitors/ParallelRecentValidator.java
@@ -57,7 +57,7 @@ public class ParallelRecentValidator extends
ParallelValidator<ParallelRecentVal
{
super(concurrency, triggerAfter, run);
this.partitionCount = partitionCount;
- this.queries = queries;
+ this.queries = Math.max(queries, 1);
this.querySelector = new QueryGenerator.TypedQueryGenerator(run.rng,
// TODO:
make query kind configurable
Surjections.enumValues(Query.QueryKind.class),
@@ -157,7 +157,7 @@ public class ParallelRecentValidator extends
ParallelValidator<ParallelRecentVal
{
this.partition_count = partition_count;
this.concurrency = concurrency;
- this.queries = queries;
+ this.queries = Math.max(queries, 1);
this.trigger_after = trigger_after;
this.modelConfiguration = model;
}
diff --git a/harry-core/src/harry/visitors/RecentValidator.java
b/harry-core/src/harry/visitors/RecentValidator.java
index a563a30..51781a2 100644
--- a/harry-core/src/harry/visitors/RecentValidator.java
+++ b/harry-core/src/harry/visitors/RecentValidator.java
@@ -64,7 +64,7 @@ public class RecentValidator implements Visitor
Model.ModelFactory modelFactory)
{
this.partitionCount = partitionCount;
- this.queries = queries;
+ this.queries = Math.max(queries, 1);
this.metricReporter = run.metricReporter;
this.pdSelector = run.pdSelector;
this.clock = run.clock;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
