aratno commented on a change in pull request #1382: URL: https://github.com/apache/cassandra/pull/1382#discussion_r791212792
########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java ########## @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; + +import harry.core.Run; +import harry.ddl.SchemaSpec; +import harry.model.OpSelectors; +import harry.operations.Relation; +import harry.operations.Query; +import harry.operations.QueryGenerator; +import harry.util.BitSet; +import org.apache.cassandra.cql3.AbstractMarker; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.SingleColumnRelation; +import org.apache.cassandra.cql3.VariableSpecifications; +import org.apache.cassandra.cql3.WhereClause; +import org.apache.cassandra.cql3.restrictions.StatementRestrictions; +import org.apache.cassandra.cql3.statements.Bound; +import org.apache.cassandra.cql3.statements.DeleteStatement; +import org.apache.cassandra.cql3.statements.StatementType; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static harry.generators.DataGenerators.UNSET_VALUE; + +public class SSTableGenerator +{ + protected final SchemaSpec schema; + protected final OpSelectors.DescriptorSelector descriptorSelector; + protected final OpSelectors.MonotonicClock clock; + protected final ColumnFamilyStore store; + protected final TableMetadata metadata; + protected final QueryGenerator rangeSelector; + + private final Set<SSTableReader> sstables = new HashSet<>(); + + private long lts = 0; + + public SSTableGenerator(Run run, + ColumnFamilyStore store) + { + this.schema = run.schemaSpec; + this.descriptorSelector = run.descriptorSelector; + this.clock = run.clock; + this.store = store; + // We assume metadata can not change over the lifetime of sstable generator + this.metadata = store.metadata.get(); + this.rangeSelector = new QueryGenerator(schema, run.pdSelector, descriptorSelector, run.rng); + store.disableAutoCompaction(); + } + + public Collection<SSTableReader> gen(int rows) + { + return gen(rows, 0); + } + + public Collection<SSTableReader> gen(int rows, int level) + { + mark(); + for (int i = 0; i < rows; i++) + { + long current = lts++; + write(current, current, current, current, true).applyUnsafe(); + if (schema.staticColumns != null) + writeStatic(current, current, current, current, true).applyUnsafe(); + } + + return flush(level); + } + + public void mark() + { + sstables.clear(); + sstables.addAll(store.getLiveSSTables()); + } + + public Collection<SSTableReader> flush(int level) + { + store.forceBlockingFlush(); + sstables.removeAll(store.getLiveSSTables()); + + for (SSTableReader reader : sstables) + { + if (reader.getSSTableLevel() != reader.getSSTableLevel()) Review comment: Typo? ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/HarryRunner.java ########## @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz; + +import java.io.File; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; +import harry.core.Configuration; +import harry.ddl.SchemaSpec; +import harry.model.sut.SystemUnderTest; +import harry.runner.Runner; + +public class HarryRunner +{ + public static final SchemaSpec schemaSpec = null; + + public static void init() + { + InJvmSut.init(); + PrintlnSutConfiguration.init(); + FaultInjectingPartitionVisitor.init(); + QueryingNoOpChecker.init(); + Configuration.registerSubtypes(Configuration.NoOpMetricReporterConfiguration.class); + Configuration.registerSubtypes(Configuration.RecentPartitionsValidatorConfiguration.class); + } + + public static void main(String[] args) throws Throwable + { + System.setProperty("log4j2.disableJmx", "true"); // setting both ways as changes between versions + System.setProperty("log4j2.disable.jmx", "true"); + System.setProperty("log4j.shutdownHookEnabled", "false"); + System.setProperty("cassandra.allow_simplestrategy", "true"); // makes easier to share OSS tests without RF limits + System.setProperty("cassandra.minimum_replication_factor", "0"); // makes easier to share OSS tests without RF limits + + System.setProperty("cassandra.disable_tcactive_openssl", "true"); + System.setProperty("relocated.shaded.io.netty.transport.noNative", "true"); + System.setProperty("org.apache.cassandra.disable_mbean_registration", "true"); + + // Initialize YAML subtypes + init(); + + harry.runner.HarryRunner runner = new harry.runner.HarryRunner() + { + public void beforeRun(Runner.TimedRunner runner) + { + runner.getRun().sut.schemaChange("CREATE KEYSPACE " + schemaSpec.keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2};"); Review comment: Shouldn't this keyspace DDL be per-test, not in the runner? ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java ########## @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; + +import harry.core.Run; +import harry.ddl.SchemaSpec; +import harry.model.OpSelectors; +import harry.operations.Relation; +import harry.operations.Query; +import harry.operations.QueryGenerator; +import harry.util.BitSet; +import org.apache.cassandra.cql3.AbstractMarker; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.SingleColumnRelation; +import org.apache.cassandra.cql3.VariableSpecifications; +import org.apache.cassandra.cql3.WhereClause; +import org.apache.cassandra.cql3.restrictions.StatementRestrictions; +import org.apache.cassandra.cql3.statements.Bound; +import org.apache.cassandra.cql3.statements.DeleteStatement; +import org.apache.cassandra.cql3.statements.StatementType; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static harry.generators.DataGenerators.UNSET_VALUE; + +public class SSTableGenerator +{ + protected final SchemaSpec schema; + protected final OpSelectors.DescriptorSelector descriptorSelector; + protected final OpSelectors.MonotonicClock clock; + protected final ColumnFamilyStore store; + protected final TableMetadata metadata; + protected final QueryGenerator rangeSelector; + + private final Set<SSTableReader> sstables = new HashSet<>(); + + private long lts = 0; + + public SSTableGenerator(Run run, + ColumnFamilyStore store) + { + this.schema = run.schemaSpec; + this.descriptorSelector = run.descriptorSelector; + this.clock = run.clock; + this.store = store; + // We assume metadata can not change over the lifetime of sstable generator + this.metadata = store.metadata.get(); + this.rangeSelector = new QueryGenerator(schema, run.pdSelector, descriptorSelector, run.rng); + store.disableAutoCompaction(); + } + + public Collection<SSTableReader> gen(int rows) + { + return gen(rows, 0); + } + + public Collection<SSTableReader> gen(int rows, int level) + { + mark(); + for (int i = 0; i < rows; i++) + { + long current = lts++; + write(current, current, current, current, true).applyUnsafe(); + if (schema.staticColumns != null) + writeStatic(current, current, current, current, true).applyUnsafe(); + } + + return flush(level); + } + + public void mark() + { + sstables.clear(); + sstables.addAll(store.getLiveSSTables()); + } + + public Collection<SSTableReader> flush(int level) + { + store.forceBlockingFlush(); + sstables.removeAll(store.getLiveSSTables()); + + for (SSTableReader reader : sstables) + { + if (reader.getSSTableLevel() != reader.getSSTableLevel()) + { + try + { + reader.descriptor.getMetadataSerializer().mutateLevel(reader.descriptor, level); + reader.reloadSSTableMetadata(); + } + catch (IOException e) + { + throw new RuntimeException("Can't mutate level", e); + } + } + } + + Set<SSTableReader> ret = new HashSet<>(sstables); + mark(); + return ret; + } + + public Mutation write(long lts, long pd, long cd, long opId, boolean withRowMarker) Review comment: Nitpicking, but I wish there was a better way to express these long-heavy signatures. If we had type aliases, we could make Lts, Pd, Cd, OpId all their own types. Then the type checker could make signatures like these more digestible, ensure we don't inflate a value we shouldn't inflate, etc. ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/test/SSTableGeneratorTest.java ########## @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz.test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import com.google.common.collect.Iterators; +import org.junit.Test; + +import harry.core.Configuration; +import harry.core.Run; +import harry.ddl.ColumnSpec; +import harry.ddl.SchemaSpec; +import harry.model.Model; +import harry.model.QuiescentChecker; +import harry.model.clock.OffsetClock; +import harry.model.sut.SystemUnderTest; +import harry.operations.Query; +import harry.visitors.GeneratingVisitor; +import harry.visitors.LtsVisitor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.distributed.fuzz.HarryRunner; +import org.apache.cassandra.distributed.fuzz.SSTableLoadingVisitor; +import org.apache.cassandra.distributed.impl.RowUtil; + +public class SSTableGeneratorTest extends CQLTester +{ + private static Configuration configuration; + + static + { + try + { + HarryRunner.init(); + configuration = HarryRunner.configuration("test/conf/harry-generic.yaml") + .unbuild() + .setClock(() -> new OffsetClock(10000L)) + .build(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private static SchemaSpec schemaSpec = new SchemaSpec(KEYSPACE, "tbl1", + Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType), + ColumnSpec.pk("pk2", ColumnSpec.int64Type)), + Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, false), + ColumnSpec.ck("ck2", ColumnSpec.int64Type, false)), + Arrays.asList(ColumnSpec.regularColumn("v1", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v2", ColumnSpec.int64Type), + ColumnSpec.regularColumn("v3", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v4", ColumnSpec.asciiType)), + Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType), + ColumnSpec.staticColumn("s2", ColumnSpec.int64Type))); + + @Test + public void testSSTableGenerator() + { + createTable(schemaSpec.compile().cql()); + Run run = configuration.unbuild() + .setSeed(1) Review comment: Nit: this seed is already specified in harry-generic.yml. ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/test/SSTableGeneratorTest.java ########## @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz.test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import com.google.common.collect.Iterators; +import org.junit.Test; + +import harry.core.Configuration; +import harry.core.Run; +import harry.ddl.ColumnSpec; +import harry.ddl.SchemaSpec; +import harry.model.Model; +import harry.model.QuiescentChecker; +import harry.model.clock.OffsetClock; +import harry.model.sut.SystemUnderTest; +import harry.operations.Query; +import harry.visitors.GeneratingVisitor; +import harry.visitors.LtsVisitor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.distributed.fuzz.HarryRunner; +import org.apache.cassandra.distributed.fuzz.SSTableLoadingVisitor; +import org.apache.cassandra.distributed.impl.RowUtil; + +public class SSTableGeneratorTest extends CQLTester +{ + private static Configuration configuration; + + static + { + try + { + HarryRunner.init(); + configuration = HarryRunner.configuration("test/conf/harry-generic.yaml") + .unbuild() + .setClock(() -> new OffsetClock(10000L)) + .build(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private static SchemaSpec schemaSpec = new SchemaSpec(KEYSPACE, "tbl1", + Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType), + ColumnSpec.pk("pk2", ColumnSpec.int64Type)), + Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, false), + ColumnSpec.ck("ck2", ColumnSpec.int64Type, false)), + Arrays.asList(ColumnSpec.regularColumn("v1", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v2", ColumnSpec.int64Type), + ColumnSpec.regularColumn("v3", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v4", ColumnSpec.asciiType)), + Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType), + ColumnSpec.staticColumn("s2", ColumnSpec.int64Type))); + + @Test + public void testSSTableGenerator() + { + createTable(schemaSpec.compile().cql()); + Run run = configuration.unbuild() + .setSeed(1) + .setSchemaProvider(new HarryRunner.FixedSchemaProviderConfiguration(schemaSpec)) + .setSUT(CqlTesterSUT::new) + .build() + .createRun(); + + + SSTableLoadingVisitor sstableVisitor = new SSTableLoadingVisitor(run, 1000); + LtsVisitor visitor = new GeneratingVisitor(run, sstableVisitor); + Set<Long> pds = new HashSet<>(); + run.tracker.onLtsStarted((lts) -> { + pds.add(run.pdSelector.pd(lts, run.schemaSpec)); + }); + for (int i = 0; i < 1000; i++) Review comment: Should this come from Configuration, via the runtime or max LTS? ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/HarryRunner.java ########## @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz; + +import java.io.File; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; +import harry.core.Configuration; +import harry.ddl.SchemaSpec; +import harry.model.sut.SystemUnderTest; +import harry.runner.Runner; + +public class HarryRunner +{ + public static final SchemaSpec schemaSpec = null; + + public static void init() + { + InJvmSut.init(); + PrintlnSutConfiguration.init(); + FaultInjectingPartitionVisitor.init(); + QueryingNoOpChecker.init(); + Configuration.registerSubtypes(Configuration.NoOpMetricReporterConfiguration.class); + Configuration.registerSubtypes(Configuration.RecentPartitionsValidatorConfiguration.class); + } + + public static void main(String[] args) throws Throwable + { + System.setProperty("log4j2.disableJmx", "true"); // setting both ways as changes between versions + System.setProperty("log4j2.disable.jmx", "true"); + System.setProperty("log4j.shutdownHookEnabled", "false"); + System.setProperty("cassandra.allow_simplestrategy", "true"); // makes easier to share OSS tests without RF limits + System.setProperty("cassandra.minimum_replication_factor", "0"); // makes easier to share OSS tests without RF limits + + System.setProperty("cassandra.disable_tcactive_openssl", "true"); + System.setProperty("relocated.shaded.io.netty.transport.noNative", "true"); + System.setProperty("org.apache.cassandra.disable_mbean_registration", "true"); Review comment: Is there any expected scenario where these would be set any other way? ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/InJvmSutBase.java ########## @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import harry.core.Configuration; +import harry.model.sut.SystemUnderTest; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICluster; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.IMessageFilters; + +public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut +{ + public static void init() + { + Configuration.registerSubtypes(InJvmSutBaseConfiguration.class); + } + + private static final Logger logger = LoggerFactory.getLogger(InJvmSutBase.class); + + // TODO: shut down properly + private final ExecutorService executor; + public final CLUSTER cluster; + private final AtomicLong cnt = new AtomicLong(); + private final AtomicBoolean isShutdown = new AtomicBoolean(false); + + public InJvmSutBase(CLUSTER cluster) + { + this(cluster, 10); + } + + public InJvmSutBase(CLUSTER cluster, int threads) + { + this.cluster = cluster; + this.executor = Executors.newFixedThreadPool(threads); + } + + public CLUSTER cluster() + { + return cluster; + } + + public boolean isShutdown() + { + return isShutdown.get(); + } + + public void shutdown() + { + assert isShutdown.compareAndSet(false, true); + + try + { + cluster.close(); + executor.shutdown(); + executor.awaitTermination(30, TimeUnit.SECONDS); + } + catch (Throwable e) + { + throw new RuntimeException(e); + } + } + + public void schemaChange(String statement) + { + cluster.schemaChange(statement); + } + + public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings) + { + return execute(statement, cl, (int) (cnt.getAndIncrement() % cluster.size() + 1), bindings); + } + + public Object[][] execute(String statement, ConsistencyLevel cl, int coordinator, Object... bindings) + { + if (isShutdown.get()) + throw new RuntimeException("Instance is shut down"); + + try + { + if (cl == ConsistencyLevel.NODE_LOCAL) + { + return cluster.get(coordinator) + .executeInternal(statement, bindings); + } + else if (statement.contains("SELECT")) Review comment: Nit: could construct Statement and check instanceof SelectStatement ########## File path: test/distributed/org/apache/cassandra/distributed/fuzz/test/SSTableGeneratorTest.java ########## @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.fuzz.test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import com.google.common.collect.Iterators; +import org.junit.Test; + +import harry.core.Configuration; +import harry.core.Run; +import harry.ddl.ColumnSpec; +import harry.ddl.SchemaSpec; +import harry.model.Model; +import harry.model.QuiescentChecker; +import harry.model.clock.OffsetClock; +import harry.model.sut.SystemUnderTest; +import harry.operations.Query; +import harry.visitors.GeneratingVisitor; +import harry.visitors.LtsVisitor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.distributed.fuzz.HarryRunner; +import org.apache.cassandra.distributed.fuzz.SSTableLoadingVisitor; +import org.apache.cassandra.distributed.impl.RowUtil; + +public class SSTableGeneratorTest extends CQLTester +{ + private static Configuration configuration; + + static + { + try + { + HarryRunner.init(); + configuration = HarryRunner.configuration("test/conf/harry-generic.yaml") + .unbuild() + .setClock(() -> new OffsetClock(10000L)) + .build(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private static SchemaSpec schemaSpec = new SchemaSpec(KEYSPACE, "tbl1", + Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType), + ColumnSpec.pk("pk2", ColumnSpec.int64Type)), + Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, false), + ColumnSpec.ck("ck2", ColumnSpec.int64Type, false)), + Arrays.asList(ColumnSpec.regularColumn("v1", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v2", ColumnSpec.int64Type), + ColumnSpec.regularColumn("v3", ColumnSpec.int32Type), + ColumnSpec.regularColumn("v4", ColumnSpec.asciiType)), + Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType), + ColumnSpec.staticColumn("s2", ColumnSpec.int64Type))); + + @Test + public void testSSTableGenerator() + { + createTable(schemaSpec.compile().cql()); + Run run = configuration.unbuild() + .setSeed(1) + .setSchemaProvider(new HarryRunner.FixedSchemaProviderConfiguration(schemaSpec)) + .setSUT(CqlTesterSUT::new) + .build() + .createRun(); + + + SSTableLoadingVisitor sstableVisitor = new SSTableLoadingVisitor(run, 1000); + LtsVisitor visitor = new GeneratingVisitor(run, sstableVisitor); + Set<Long> pds = new HashSet<>(); + run.tracker.onLtsStarted((lts) -> { + pds.add(run.pdSelector.pd(lts, run.schemaSpec)); + }); + for (int i = 0; i < 1000; i++) + visitor.visit(); + + sstableVisitor.forceFlush(0); + + Model checker = new QuiescentChecker(run); + for (Long pd : pds) + checker.validate(Query.selectPartition(run.schemaSpec, pd, false)); Review comment: Would be nice if this returned some ModelResult so we could check that number of rows validated per partition met our expectations. ########## File path: test/conf/harry-generic.yaml ########## @@ -0,0 +1,76 @@ +seed: 1 + +# Default schema provider generates random schema +schema_provider: + default: {} + +drop_schema: false +create_schema: true +truncate_table: false + +clock: + approximate_monotonic: + history_size: 7300 + epoch_length: 1 + epoch_time_unit: "SECONDS" + +system_under_test: + println: {} + +partition_descriptor_selector: + default: + window_size: 100 + slide_after_repeats: 10 + +clustering_descriptor_selector: + default: + modifications_per_lts: + type: "constant" + constant: 2 + rows_per_modification: + type: "constant" + constant: 2 + operation_kind_weights: + DELETE_RANGE: 1 + DELETE_SLICE: 1 + DELETE_ROW: 1 + DELETE_COLUMN: 1 + DELETE_PARTITION: 1 + DELETE_COLUMN: 1 + INSERT: 50 + UPDATE: 50 + DELETE_COLUMN_WITH_STATICS: 1 + INSERT_WITH_STATICS: 1 + UPDATE_WITH_STATICS: 1 + column_mask_bitsets: null + max_partition_size: 100 + +data_tracker: + default: + max_seen_lts: -1 + max_complete_lts: -1 + +runner: + sequential: + run_time: 60 + run_time_unit: "MINUTES" + visitors: + - logging: + row_visitor: + mutating: {} + - sampler: + trigger_after: 100000 + sample_partitions: 10 + - validate_recent_partitions: + partition_count: 5 + trigger_after: 10000 + model: + querying_no_op_checker: {} + - validate_all_partitions: + concurrency: 5 + trigger_after: 10000 + model: + querying_no_op_checker: {} + +metric_reporter: + no_op: {} Review comment: Nit: move into a Configuration instance instead of keeping in static yml form, to avoid the `unbuild()` calls elsewhere. Could use with-copy methods (such as `withDropSchema(false)`) to make the relationship between the base Configuration and others more clear. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
