maedhroz commented on a change in pull request #1382:
URL: https://github.com/apache/cassandra/pull/1382#discussion_r791091248
##########
File path: test/conf/harry-generic.yaml
##########
@@ -0,0 +1,76 @@
+seed: 1
Review comment:
nit: Need to add the Apache license header or rat will complain on build
##########
File path: test/conf/harry-generic.yaml
##########
@@ -0,0 +1,76 @@
+seed: 1
+
+# Default schema provider generates random schema
+schema_provider:
+ default: {}
+
+drop_schema: false
+create_schema: true
+truncate_table: false
+
+clock:
+ approximate_monotonic:
+ history_size: 7300
+ epoch_length: 1
+ epoch_time_unit: "SECONDS"
+
+system_under_test:
+ println: {}
+
+partition_descriptor_selector:
+ default:
+ window_size: 100
+ slide_after_repeats: 10
+
+clustering_descriptor_selector:
+ default:
+ modifications_per_lts:
+ type: "constant"
+ constant: 2
+ rows_per_modification:
+ type: "constant"
+ constant: 2
+ operation_kind_weights:
+ DELETE_RANGE: 1
+ DELETE_SLICE: 1
+ DELETE_ROW: 1
+ DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN: 1
Review comment:
nit: `DELETE_COLUMN` is in here twice
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/InJvmSutBase.java
##########
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import harry.core.Configuration;
+import harry.model.sut.SystemUnderTest;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IMessage;
+import org.apache.cassandra.distributed.api.IMessageFilters;
+
+public class InJvmSutBase<NODE extends IInstance, CLUSTER extends
ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
+{
+ public static void init()
+ {
+ Configuration.registerSubtypes(InJvmSutBaseConfiguration.class);
+ }
+
+ private static final Logger logger =
LoggerFactory.getLogger(InJvmSutBase.class);
+
+ // TODO: shut down properly
+ private final ExecutorService executor;
+ public final CLUSTER cluster;
+ private final AtomicLong cnt = new AtomicLong();
+ private final AtomicBoolean isShutdown = new AtomicBoolean(false);
+
+ public InJvmSutBase(CLUSTER cluster)
+ {
+ this(cluster, 10);
+ }
+
+ public InJvmSutBase(CLUSTER cluster, int threads)
+ {
+ this.cluster = cluster;
+ this.executor = Executors.newFixedThreadPool(threads);
+ }
+
+ public CLUSTER cluster()
+ {
+ return cluster;
+ }
+
+ public boolean isShutdown()
+ {
+ return isShutdown.get();
+ }
+
+ public void shutdown()
Review comment:
nit: Throw an `@Override` on this and `isShutdown()`
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/InJvmSutBase.java
##########
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import harry.core.Configuration;
+import harry.model.sut.SystemUnderTest;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IMessage;
+import org.apache.cassandra.distributed.api.IMessageFilters;
+
+public class InJvmSutBase<NODE extends IInstance, CLUSTER extends
ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
+{
+ public static void init()
+ {
+ Configuration.registerSubtypes(InJvmSutBaseConfiguration.class);
+ }
+
+ private static final Logger logger =
LoggerFactory.getLogger(InJvmSutBase.class);
+
+ // TODO: shut down properly
Review comment:
Anything other than what's already being done in `shutdown()`?
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java
##########
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+
+import harry.core.Run;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
+import harry.operations.Relation;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
+import harry.util.BitSet;
+import org.apache.cassandra.cql3.AbstractMarker;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.SingleColumnRelation;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.cql3.statements.Bound;
+import org.apache.cassandra.cql3.statements.DeleteStatement;
+import org.apache.cassandra.cql3.statements.StatementType;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static harry.generators.DataGenerators.UNSET_VALUE;
+
+public class SSTableGenerator
+{
+ protected final SchemaSpec schema;
+ protected final OpSelectors.DescriptorSelector descriptorSelector;
+ protected final OpSelectors.MonotonicClock clock;
+ protected final ColumnFamilyStore store;
+ protected final TableMetadata metadata;
+ protected final QueryGenerator rangeSelector;
+
+ private final Set<SSTableReader> sstables = new HashSet<>();
+
+ private long lts = 0;
+
+ public SSTableGenerator(Run run,
+ ColumnFamilyStore store)
+ {
+ this.schema = run.schemaSpec;
+ this.descriptorSelector = run.descriptorSelector;
+ this.clock = run.clock;
+ this.store = store;
+ // We assume metadata can not change over the lifetime of sstable
generator
+ this.metadata = store.metadata.get();
+ this.rangeSelector = new QueryGenerator(schema, run.pdSelector,
descriptorSelector, run.rng);
+ store.disableAutoCompaction();
+ }
+
+ public Collection<SSTableReader> gen(int rows)
+ {
+ return gen(rows, 0);
+ }
+
+ public Collection<SSTableReader> gen(int rows, int level)
+ {
+ mark();
+ for (int i = 0; i < rows; i++)
+ {
+ long current = lts++;
+ write(current, current, current, current, true).applyUnsafe();
+ if (schema.staticColumns != null)
+ writeStatic(current, current, current, current,
true).applyUnsafe();
+ }
+
+ return flush(level);
+ }
+
+ public void mark()
+ {
+ sstables.clear();
+ sstables.addAll(store.getLiveSSTables());
+ }
+
+ public Collection<SSTableReader> flush(int level)
+ {
+ store.forceBlockingFlush();
+ sstables.removeAll(store.getLiveSSTables());
+
+ for (SSTableReader reader : sstables)
+ {
+ if (reader.getSSTableLevel() != reader.getSSTableLevel())
Review comment:
Related: Is there any case where we aren't flushing to L0? Just a little
worries that the API we have here might allow flushing to L1+ with overlapping
ranges...
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java
##########
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.util.Collection;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaSpec;
+import harry.model.clock.OffsetClock;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public abstract class FuzzTestBase extends TestBaseImpl
+{
+ protected static Configuration configuration;
+ public static final int RF = 2;
+ static
+ {
+ try
+ {
+ HarryRunner.init();
+ configuration =
HarryRunner.configuration("test/conf/harry-generic.yaml");
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected static Cluster cluster;
+
+ @BeforeClass
+ public static void beforeClassOverride() throws Throwable
+ {
+ cluster = Cluster.build(2)
+ .start();
+
+ init(cluster, RF);
+ }
+
+ @AfterClass
+ public static void afterClass()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ public void reset()
+ {
+ cluster.schemaChange("DROP KEYSPACE IF EXISTS " + KEYSPACE);
+ init(cluster, RF);
+ }
+
+ public static void generateTables(SchemaSpec schemaSpec, int number)
Review comment:
...and of course `SSTableWithLargePartition` becomes unused if we remove
this.
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/HarryRunner.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.io.File;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.sut.SystemUnderTest;
+import harry.runner.Runner;
+
+public class HarryRunner
+{
+ public static final SchemaSpec schemaSpec = null;
+
+ public static void init()
+ {
+ InJvmSut.init();
+ PrintlnSutConfiguration.init();
+ FaultInjectingPartitionVisitor.init();
+ QueryingNoOpChecker.init();
+
Configuration.registerSubtypes(Configuration.NoOpMetricReporterConfiguration.class);
+
Configuration.registerSubtypes(Configuration.RecentPartitionsValidatorConfiguration.class);
+ }
+
+ public static void main(String[] args) throws Throwable
+ {
+ System.setProperty("log4j2.disableJmx", "true"); // setting both ways
as changes between versions
+ System.setProperty("log4j2.disable.jmx", "true");
+ System.setProperty("log4j.shutdownHookEnabled", "false");
+ System.setProperty("cassandra.allow_simplestrategy", "true"); // makes
easier to share OSS tests without RF limits
+ System.setProperty("cassandra.minimum_replication_factor", "0"); //
makes easier to share OSS tests without RF limits
+
+ System.setProperty("cassandra.disable_tcactive_openssl", "true");
+ System.setProperty("relocated.shaded.io.netty.transport.noNative",
"true");
+ System.setProperty("org.apache.cassandra.disable_mbean_registration",
"true");
+
+ // Initialize YAML subtypes
+ init();
+
+ harry.runner.HarryRunner runner = new harry.runner.HarryRunner()
+ {
+ public void beforeRun(Runner.TimedRunner runner)
+ {
+ runner.getRun().sut.schemaChange("CREATE KEYSPACE " +
schemaSpec.keyspace + " WITH replication = {'class': 'SimpleStrategy',
'replication_factor': 2};");
Review comment:
Expanding on this, if you try to run the `main()` method, the
`schemaSpec` is indeed `null`, and you get a NPE when `createRun()` tries to
create a `SchemeSpec` from the `SchemaProviderConfiguration`...
```
Exception in thread "main" java.lang.NullPointerException
at harry.core.Configuration.createRun(Configuration.java:238)
at harry.core.Configuration.createRunner(Configuration.java:255)
at harry.core.Configuration.createRunner(Configuration.java:213)
at harry.runner.HarryRunner.run(HarryRunner.java:63)
at
org.apache.cassandra.distributed.fuzz.HarryRunner.main(HarryRunner.java:71)
```
##########
File path:
test/distributed/org/apache/cassandra/distributed/fuzz/HarryRunner.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.fuzz;
+
+import java.io.File;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.sut.SystemUnderTest;
+import harry.runner.Runner;
+
+public class HarryRunner
Review comment:
super nit: Maybe we could name this something like `HarryRunnerExample`
to avoid any confusion w/ `harry.runner.HarryRunner`. (Seems like the purpose
here was just to demonstrate how a standalone harry run could be constructed.)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
