smiklosovic commented on code in PR #3374: URL: https://github.com/apache/cassandra/pull/3374#discussion_r1838592301
########## test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java: ########## @@ -0,0 +1,880 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.fuzz.snapshots; + +import java.io.IOException; +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; + +import accord.utils.Property.Command; +import accord.utils.RandomSource; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableTriConsumer; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.fuzz.snapshots.SnapshotsTest.State.TestSnapshot; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.utils.Generators; +import org.apache.cassandra.utils.Pair; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.impl.JavaRandom; + +import static accord.utils.Property.commands; +import static accord.utils.Property.stateful; +import static com.google.common.collect.Sets.difference; +import static java.lang.String.format; +import static org.junit.Assert.assertEquals; + +public class SnapshotsTest +{ + @Test + public void fuzzySnapshotsTest() + { + stateful() + .withExamples(1) + .withSteps(1000) + .check(commands(() -> State::new, state -> new SnapshotsSut(state).start()) + .add(10, new CreateKeyspaceCommand()) + .add(5, new DropKeyspaceCommand()) + .add(40, new CreateTableCommand()) + .add(15, new DropTableCommand()) + .add(500, new TakeSnapshotCommand()) + .add(50, new ClearSnapshotCommand()) + .destroyState(State::destroy) + .destroySut(SnapshotsSut::stop) + .build()); + } + + public static class State implements Serializable + { + public Map<String, List<String>> schema; + public Set<TestSnapshot> snapshots = new HashSet<>(); + public Set<String> droppedSnapshots = new HashSet<>(); + public Set<String> truncatedSnapshots = new HashSet<>(); + + public final RandomSource rs; + public final RandomnessSource randomnessSource; + + public State(RandomSource rs) + { + this.rs = rs; + this.randomnessSource = new JavaRandom(rs.asJdkRandom()); + } + + public State populate(SnapshotsSut sut) + { + this.schema = sut.getSchema(); + this.snapshots = sut.getSnapshots(); + this.droppedSnapshots = sut.getSnapshotsOfDroppedTables(); + this.truncatedSnapshots = sut.getSnapshotsOfTruncatedTables(); + return this; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + State state = (State) o; + return Objects.equals(schema, state.schema) && + Objects.equals(snapshots, state.snapshots) && + Objects.equals(droppedSnapshots, state.droppedSnapshots) && + Objects.equals(truncatedSnapshots, state.truncatedSnapshots); + } + + @Override + public int hashCode() + { + return Objects.hash(schema, snapshots, droppedSnapshots, truncatedSnapshots); + } + + @Override + public String toString() + { + return "State{" + + "schema=" + schema + + ", snapshots=" + snapshots + + ", droppedSnapshots=" + droppedSnapshots + + ", truncatedSnapshots=" + truncatedSnapshots + + '}'; + } + + public Optional<String> pickRandomKeyspace() + { + if (schema.keySet().isEmpty()) + return Optional.empty(); + else + return Optional.of(rs.pick(schema.keySet())); + } + + public Pair<String, String> pickOrCreateRandomTable() + { + String keyspace = pickRandomKeyspace().orElseGet(this::createRandomKeyspaceIdentifier); + + String tableName; + List<String> tablesOfRandomKeyspace = schema.get(keyspace); + if (tablesOfRandomKeyspace == null || tablesOfRandomKeyspace.isEmpty()) + tableName = createRandomTableIdentifier(); + else + tableName = rs.pick(tablesOfRandomKeyspace); + + if (schema.get(keyspace) == null || (tablesOfRandomKeyspace == null || tablesOfRandomKeyspace.isEmpty())) + addTable(keyspace, tableName); + + return Pair.create(keyspace, tableName); + } + + public String addRandomKeyspace() + { + String keyspace = createRandomKeyspaceIdentifier(); + addKeyspace(keyspace); + return keyspace; + } + + public Optional<Pair<String, String>> addRandomTable() + { + // no keyspace yet, create one + if (schema.keySet().isEmpty()) + addRandomKeyspace(); + + Optional<String> randomKeyspace = pickRandomKeyspace(); + + if (randomKeyspace.isPresent()) + { + String randomTableName = createRandomTableIdentifier(); + + addTable(randomKeyspace.get(), randomTableName); + return Optional.of(Pair.create(randomKeyspace.get(), randomTableName)); + } + + return Optional.empty(); + } + + public Optional<Pair<String, List<String>>> removeRandomKeyspace() + { + Optional<String> randomKeyspace = pickRandomKeyspace(); + return randomKeyspace.map(this::removeKeyspace); + } + + public Optional<Pair<String, String>> removeRandomTable() + { + Optional<String> randomKeyspace = pickRandomKeyspace(); + + if (randomKeyspace.isPresent()) + { + List<String> tablesOfRandomKeyspace = schema.get(randomKeyspace.get()); + + if (!tablesOfRandomKeyspace.isEmpty()) + { + String randomlyPickedTable = rs.pick(tablesOfRandomKeyspace); + removeTable(randomKeyspace.get(), randomlyPickedTable); + + return Optional.of(Pair.create(randomKeyspace.get(), randomlyPickedTable)); + } + } + + return Optional.empty(); + } + + public void addKeyspace(String keyspaceName) + { + schema.put(keyspaceName, new ArrayList<>()); + } + + public Pair<String, List<String>> removeKeyspace(String keyspaceName) + { + List<String> removedTables = schema.remove(keyspaceName); + return Pair.create(keyspaceName, removedTables); + } + + public void addTable(String keyspaceName, String tableName) + { + List<String> tables = schema.get(keyspaceName); + if (tables == null) + tables = new ArrayList<>(); + + tables.add(tableName); + schema.put(keyspaceName, tables); + } + + public void removeTable(String keyspace, String table) + { + List<String> tables = schema.get(keyspace); + if (tables == null) + return; + + tables.remove(table); + } + + public String addSnapshot(String keyspace, String table) + { + String randomSnapshotIdentifier = createRandomSnapshotIdentifier(); + addSnapshot(new TestSnapshot(keyspace, table, randomSnapshotIdentifier)); + + return randomSnapshotIdentifier; + } + + public void addDroppedSnapshot(String keyspace, String table) + { + droppedSnapshots.add(keyspace + '.' + table); + } + + public void addTruncatedSnapshot(String keyspace, String table) + { + truncatedSnapshots.add(keyspace + '.' + table); + } + + public void addSnapshot(TableSnapshot snapshot) + { + snapshots.add(new TestSnapshot(snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag())); + } + + public void destroy() + { + snapshots.clear(); + droppedSnapshots.clear(); + truncatedSnapshots.clear(); + schema.clear(); + } + + // we just need something to store in the state and whole thing is too much and not necessary + public static class TestSnapshot extends TableSnapshot implements Comparable<TestSnapshot> + { + public TestSnapshot(String keyspaceName, String tableName, String tag) + { + this(keyspaceName, tableName, UUID.randomUUID(), tag, null, null, null, false); + } + + public TestSnapshot(String keyspaceName, String tableName, UUID tableId, String tag, + Instant createdAt, Instant expiresAt, Set<File> snapshotDirs, boolean ephemeral) + { + super(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral); + } + + @Override + public long getManifestsSize() + { + return 0; + } + + @Override + public long getSchemasSize() + { + return 0; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableSnapshot snapshot = (TableSnapshot) o; + return Objects.equals(getKeyspaceName(), snapshot.getKeyspaceName()) && + Objects.equals(getTableName(), snapshot.getTableName()) && + Objects.equals(getTag(), snapshot.getTag()); + } + + @Override + public int hashCode() + { + return Objects.hash(getKeyspaceName(), getTableName(), getTag()); + } + + @Override + public String toString() + { + return "TestSnapshot{" + + "keyspaceName='" + getKeyspaceName() + '\'' + + ", tableName='" + getTableName() + '\'' + + ", tag='" + getTag() + '\'' + + '}'; + } + + @Override + public int compareTo(TestSnapshot o) + { + return toString().compareTo(o.toString()); + } + } + + + private String createRandomKeyspaceIdentifier() + { + return trimIfNecessary("keyspace_" + getRandomString()); + } + + private String createRandomSnapshotIdentifier() + { + return trimIfNecessary("snapshot_" + getRandomString()); + } + + private String createRandomTableIdentifier() + { + return trimIfNecessary("table_" + getRandomString()); + } + + private String getRandomString() + { + return Generators.IDENTIFIER_GEN.generate(new JavaRandom(rs.asJdkRandom())).toLowerCase(); + } + + private String trimIfNecessary(String maybeTooLongString) + { + if (maybeTooLongString.length() <= 48) + return maybeTooLongString; + + return maybeTooLongString.substring(0, 48); + } + } + + public static class SnapshotsSut implements Serializable + { + private final Cluster.Builder builder = Cluster.build(1).withConfig(c -> c.with(Feature.values())); + private Cluster cluster; + public State state; + + public SnapshotsSut(State snapshotsState) + { + this.state = snapshotsState; + } + + public IInvokableInstance getNode() + { + if (cluster == null) + throw new RuntimeException("not started yet"); + + return cluster.get(1); + } + + public NodeToolResult nodetool(String... nodetoolArgs) + { + return getNode().nodetoolResult(nodetoolArgs); + } + + public SnapshotsSut start() + { + if (cluster != null) + return null; + + try + { + cluster = builder.start(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + + state.schema = getSchema(); + + return this; + } + + public void stop() + { + if (cluster != null) + cluster.close(); + } + + public Map<String, List<String>> getSchema() + { + IIsolatedExecutor.SerializableCallable<Map<String, List<String>>> callable = () -> { + Keyspaces keyspaces = Schema.instance.distributedKeyspaces(); + + Map<String, List<String>> keyspacesWithTables = new HashMap<>(); + + for (KeyspaceMetadata ksm : keyspaces) + { + if (ksm.name.startsWith("system_")) + continue; + + List<String> tables = new ArrayList<>(); + for (TableMetadata tmd : ksm.tables) + tables.add(tmd.name); + + keyspacesWithTables.put(ksm.name, tables); + } + + return keyspacesWithTables; + }; + + return getNode().callOnInstance(callable); + } + + public Set<String> getKeyspaces() + { + return getSchema().keySet(); + } + + private Set<String> getSnapshotsInternal(String prefix) + { + IIsolatedExecutor.SerializableFunction<String, Set<String>> callable = (p) -> { + Set<String> snapshots = new HashSet<>(); + for (TableSnapshot snapshot : SnapshotManager.instance.getSnapshots(snapshot -> snapshot.getTag().startsWith(p))) + snapshots.add(format("%s.%s", snapshot.getKeyspaceName(), snapshot.getTableName())); + + return snapshots; + }; + + return getNode().applyOnInstance(callable, prefix); + } + + public Set<String> getSnapshotsOfDroppedTables() + { + return getSnapshotsInternal("dropped-"); + } + + public Set<String> getSnapshotsOfTruncatedTables() + { + return getSnapshotsInternal("truncated-"); + } + + // these are not "dropped" nor "truncated" + public Set<TestSnapshot> getSnapshots() + { + Set<TestSnapshot> existingSnapshots = new HashSet<>(); + + IIsolatedExecutor.SerializableCallable<List<String>> callable = () -> { + List<String> snapshots = new ArrayList<>(); + for (TableSnapshot snapshot : SnapshotManager.instance.getSnapshots(p -> { + String tag = p.getTag(); + return !tag.startsWith("dropped-") && !tag.startsWith("truncated-"); + })) + { + snapshots.add(format("%s.%s.%s", snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag())); + } + + return snapshots; + }; + + for (String tableSnapshot : getNode().callOnInstance(callable)) + { + String[] components = tableSnapshot.split("\\."); + existingSnapshots.add(new TestSnapshot(components[0], + components[1], + components[2])); + } + + return existingSnapshots; + } + + public void takeSnapshots(Set<TestSnapshot> snapshotsToTake) + { + for (TestSnapshot toTake : snapshotsToTake) + getNode().acceptsOnInstance((SerializableTriConsumer<String, String, String>) (tag, ks, tb) -> SnapshotManager.instance.takeSnapshot(tag, ks, tb)) + .accept(toTake.getTag(), toTake.getKeyspaceName(), toTake.getTableName()); + } + } + + public static class ClearSnapshotCommand implements Command<State, SnapshotsSut, State> + { + private State state; + + @Override + public State apply(State state) throws Throwable + { + List<Integer> toShuffle = new ArrayList<>(); + toShuffle.add(1); + toShuffle.add(2); + toShuffle.add(3); + Collections.shuffle(toShuffle, state.rs.asJdkRandom()); + + for (int i = 0; i < 3; i++) + { + boolean picked = false; + switch (toShuffle.get(i)) + { + case 1: + if (!state.truncatedSnapshots.isEmpty()) + { + state.truncatedSnapshots.remove(state.rs.pick(state.truncatedSnapshots)); + picked = true; + } + break; + case 2: + if (!state.droppedSnapshots.isEmpty()) + { + state.droppedSnapshots.remove(state.rs.pick(state.droppedSnapshots)); + picked = true; + } + break; + case 3: + if (!state.snapshots.isEmpty()) + { + TestSnapshot pickedSnapshot = state.rs.pick(state.snapshots); + state.snapshots.remove(pickedSnapshot); + } + break; + } + if (picked) + break; + } + + this.state = state; + return state; + } + + @Override + public State run(SnapshotsSut sut) throws Throwable + { + Set<TestSnapshot> existingNormal = sut.getSnapshots(); + Set<String> existingDropped = sut.getSnapshotsOfDroppedTables(); + Set<String> existingTruncated = sut.getSnapshotsOfTruncatedTables(); + + Set<TestSnapshot> normalToBe = sut.state.snapshots; + Set<String> droppedToBe = sut.state.droppedSnapshots; + Set<String> truncatedToBe = sut.state.truncatedSnapshots; + + clearSnapshot(sut, difference(existingNormal, normalToBe)); + clearSnapshot(sut, difference(existingTruncated, truncatedToBe), "truncated-"); + clearSnapshot(sut, difference(existingDropped, droppedToBe), "dropped-"); + + return new State(sut.state.rs).populate(sut); + } + + @Override + public void checkPostconditions(State state, State expected, SnapshotsSut sut, State actual) throws Throwable + { + Assert.assertEquals(expected, actual); + } + + private void clearSnapshot(SnapshotsSut sut, Set<String> toRemove, String prefix) + { + for (String snapshot : toRemove) + { + String[] split = snapshot.split("\\."); + sut.getNode().acceptsOnInstance((SerializableTriConsumer<String, String, String>) (ks, tb, pref) -> + { + SnapshotManager.instance.clearSnapshot(s -> s.getKeyspaceName().equals(ks) && + s.getTableName().equals(tb) && + s.getTag().startsWith(pref)); + }).accept(split[0], split[1], prefix); + } + } + + private void clearSnapshot(SnapshotsSut sut, Set<TestSnapshot> toRemove) + { + for (TestSnapshot snapshot : toRemove) + { + sut.getNode() + .acceptsOnInstance((SerializableTriConsumer<String, String, String>) (ks, tb, tag) -> SnapshotManager.instance.clearSnapshot(ks, tb, tag)) + .accept(snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag()); + } + } + + @Override + public String toString() + { + return "ClearSnapshotCommand{" + + "state=" + state + + '}'; + } + } + + public static class TakeSnapshotCommand implements Command<State, SnapshotsSut, State> + { + private State state; + + @Override + public State apply(State state) + { + Pair<String, String> randomTable = state.pickOrCreateRandomTable(); + state.addSnapshot(randomTable.left, randomTable.right); + this.state = state; + return state; + } + + @Override + public State run(SnapshotsSut sut) + { + // see what snapshots are to be taken + Set<TestSnapshot> existing = sut.getSnapshots(); + Set<TestSnapshot> toBe = sut.state.snapshots; + + Set<TestSnapshot> snapshotsToTake = difference(toBe, existing); + + if (snapshotsToTake.isEmpty()) + return sut.state; + + // create missing tables if any + CreateKeyspaceCommand.createKeypaces(sut); + CreateTableCommand.createTables(sut); + + sut.takeSnapshots(snapshotsToTake); + + return new State(sut.state.rs).populate(sut); + } + + @Override + public void checkPostconditions(State state, State expected, SnapshotsSut sut, State actual) throws Throwable + { + assertEquals(expected, actual); + } + + @Override + public String toString() + { + return "TakeSnapshotCommand{" + + "state=" + state + + '}'; + } + } + + public static class CreateTableCommand implements Command<State, SnapshotsSut, State> + { + private State state; + + @Override + public State apply(State state) + { + state.addRandomTable(); + this.state = state; + return state; + } + + @Override + public State run(SnapshotsSut sut) + { + // create keyspaces if create table command created a table where no keyspaces were created yet + CreateKeyspaceCommand.createKeypaces(sut); + CreateTableCommand.createTables(sut); + return new State(sut.state.rs).populate(sut); + } + + @Override + public void checkPostconditions(State state, State expected, SnapshotsSut sut, State actual) throws Throwable + { + assertEquals(expected, actual); + } + + public static void createTables(SnapshotsSut sut) + { + Map<String, List<String>> existingSchema = sut.getSchema(); + Map<String, List<String>> schemaToBe = sut.state.schema; + + MapDifference<String, List<String>> difference = Maps.difference(schemaToBe, existingSchema); + + for (Map.Entry<String, MapDifference.ValueDifference<List<String>>> entry : difference.entriesDiffering().entrySet()) + { + String keyspaceName = entry.getKey(); + MapDifference.ValueDifference<List<String>> tableNames = entry.getValue(); + for (String table : difference(new HashSet<>(tableNames.leftValue()), new HashSet<>(tableNames.rightValue()))) + { + sut.getNode().executeInternal(format("CREATE TABLE %s.%s (id uuid primary key, val uuid);", keyspaceName, table)); + // create between 1 and 10 sstables per table + for (int i = 0; i < sut.state.rs.nextInt(1, 10); i++) + { + sut.getNode().executeInternal(format("INSERT INTO %s.%s (id, val) VALUES (%s, %s)", keyspaceName, table, UUID.randomUUID(), UUID.randomUUID())); + sut.getNode().nodetool("flush", keyspaceName, table); + } + } + } + } + + @Override + public String toString() + { + return "CreateTableCommand{" + + "state=" + state + + '}'; + } + } + + public static class DropTableCommand implements Command<State, SnapshotsSut, State> + { + private State state; + + @Override + public State apply(State state) + { + // if we drop a table, it will make a snapshot with "dropped-" prefix + // hence it will be among snapshot as well, however we do not know its name + // in advance because it contains timestamp in its name produced by Cassandra + // hence we can not add it among "normal" snapshots, hence special "droppedSnapshots" set. + state.removeRandomTable().ifPresent(p -> state.addDroppedSnapshot(p.left, p.right)); + this.state = state; + return state; + } + + @Override + public State run(SnapshotsSut sut) + { + Map<String, List<String>> existingSchema = sut.getSchema(); + Map<String, List<String>> schemaToBe = sut.state.schema; + + MapDifference<String, List<String>> difference = Maps.difference(schemaToBe, existingSchema); + + for (Map.Entry<String, MapDifference.ValueDifference<List<String>>> entry : difference.entriesDiffering().entrySet()) + { + String keyspaceName = entry.getKey(); + MapDifference.ValueDifference<List<String>> tableNames = entry.getValue(); + + Set<String> left = new HashSet<>(tableNames.leftValue()); + Set<String> right = new HashSet<>(tableNames.rightValue()); + + Sets.SetView<String> diff = difference(right, left); + + for (String table : diff) + sut.getNode().executeInternal(format("DROP TABLE %s.%s;", keyspaceName, table)); + } + + return new State(sut.state.rs).populate(sut); + } + + @Override + public void checkPostconditions(State state, State expected, SnapshotsSut sut, State actual) throws Throwable + { + assertEquals(expected, actual); + } + + @Override + public String toString() + { + return "DropTableCommand{" + + "state=" + state + + '}'; + } + } + + public static class CreateKeyspaceCommand implements Command<State, SnapshotsSut, State> + { + private State state; Review Comment: not applicable anymore -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
