This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch semi-integrated-burn-test-rebased in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit fb5d014247e0ca7f82f89b031fac4918d4fcb512 Author: Alex Petrov <[email protected]> AuthorDate: Fri Nov 15 20:25:52 2024 +0100 Fixes --- .../main-test/java/accord/burn/BurnTestBase.java | 7 +- .../main-test/java/accord/impl/basic/Cluster.java | 2 +- .../java/accord/impl/basic/VerifyingJournal.java | 94 ++++++++++++++++++++++ .../java/accord/impl/list/ListResult.java | 2 +- .../main/java/accord/coordinate/PersistTxn.java | 1 - .../java/accord/impl/DurabilityScheduling.java | 11 +-- .../src/main/java/accord/local/Command.java | 8 +- .../src/main/java/accord/messages/CheckStatus.java | 22 +++-- 8 files changed, 115 insertions(+), 32 deletions(-) diff --git a/accord-core/src/main-test/java/accord/burn/BurnTestBase.java b/accord-core/src/main-test/java/accord/burn/BurnTestBase.java index 0f5a34c9..8f2389f1 100644 --- a/accord-core/src/main-test/java/accord/burn/BurnTestBase.java +++ b/accord-core/src/main-test/java/accord/burn/BurnTestBase.java @@ -92,7 +92,6 @@ import accord.utils.RandomSource; import accord.utils.Utils; import accord.utils.async.AsyncExecutor; import accord.utils.async.TimeoutUtils; -import accord.verify.CompositeVerifier; import accord.verify.StrictSerializabilityVerifier; import accord.verify.Verifier; import org.agrona.collections.Int2ObjectHashMap; @@ -302,7 +301,7 @@ public class BurnTestBase f2.get(); } - void burn(RandomSource random, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int operations, int concurrency, PendingQueue pendingQueue, Function<Id, Journal> journalFactory) + protected void burn(RandomSource random, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int operations, int concurrency, PendingQueue pendingQueue, Function<Id, Journal> journalFactory) { List<Throwable> failures = Collections.synchronizedList(new ArrayList<>()); AtomicLong progress = new AtomicLong(); @@ -520,7 +519,7 @@ public class BurnTestBase protected Verifier createVerifier(String prefix, int keyCount) { - return new StrictSerializabilityVerifier(prefix, keyCount); + return new StrictSerializabilityVerifier(prefix, keyCount); } public static void main(String[] args) @@ -660,7 +659,7 @@ public class BurnTestBase } } - private static List<Id> generateIds(boolean clients, int count) + protected static List<Id> generateIds(boolean clients, int count) { List<Id> ids = new ArrayList<>(); for (int i = 1; i <= count ; ++i) diff --git a/accord-core/src/main-test/java/accord/impl/basic/Cluster.java b/accord-core/src/main-test/java/accord/impl/basic/Cluster.java index bd2fb921..e9c9d9ed 100644 --- a/accord-core/src/main-test/java/accord/impl/basic/Cluster.java +++ b/accord-core/src/main-test/java/accord/impl/basic/Cluster.java @@ -716,7 +716,7 @@ public class Cluster if (!store.unsafeCommands().containsKey(txnId)) { Command beforeCommand = before.get(txnId); - if (beforeCommand.saveStatus() == SaveStatus.Erased) + if (beforeCommand.saveStatus() == SaveStatus.Erased || beforeCommand.saveStatus() == SaveStatus.Uninitialised) continue; if (store.unsafeGetRedundantBefore().min(beforeCommand.participants().owns(), RedundantBefore.Entry::shardRedundantBefore).compareTo(txnId) > 0) diff --git a/accord-core/src/main-test/java/accord/impl/basic/VerifyingJournal.java b/accord-core/src/main-test/java/accord/impl/basic/VerifyingJournal.java new file mode 100644 index 00000000..0c2f80bd --- /dev/null +++ b/accord-core/src/main-test/java/accord/impl/basic/VerifyingJournal.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.impl.basic; + +import java.util.NavigableMap; + +import accord.api.Journal; +import accord.local.Command; +import accord.local.CommandStores; +import accord.local.DurableBefore; +import accord.local.RedundantBefore; +import accord.primitives.Ranges; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.utils.Invariants; + +public class VerifyingJournal implements Journal +{ + private final Journal model; + private final Journal sut; + + public VerifyingJournal(Journal model, Journal sut) + { + this.model = model; + this.sut = sut; + } + + public Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore) + { + Command res = sut.loadCommand(commandStoreId, txnId, redundantBefore, durableBefore); + Command model = this.model.loadCommand(commandStoreId, txnId, redundantBefore, durableBefore); + Invariants.checkState(res.equals(model)); + return res; + } + + public void saveCommand(int store, CommandUpdate update, Runnable onFlush) + { + model.saveCommand(store, update, null); + sut.saveCommand(store, update, onFlush); + } + + public void purge(CommandStores commandStores) + { + model.purge(commandStores); + sut.purge(commandStores); + } + + public void replay(CommandStores commandStores) + { + sut.replay(commandStores); + } + + // TODO (required): model loading + public RedundantBefore loadRedundantBefore(int commandStoreId) + { + return sut.loadRedundantBefore(commandStoreId); + } + + public NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int commandStoreId) + { + return sut.loadBootstrapBeganAt(commandStoreId); + } + + public NavigableMap<Timestamp, Ranges> loadSafeToRead(int commandStoreId) + { + return sut.loadSafeToRead(commandStoreId); + } + + public CommandStores.RangesForEpoch.Snapshot loadRangesForEpoch(int commandStoreId) + { + return sut.loadRangesForEpoch(commandStoreId); + } + + public void saveStoreState(int store, FieldUpdates fieldUpdates, Runnable onFlush) + { + sut.saveStoreState(store, fieldUpdates, onFlush); + } +} diff --git a/accord-core/src/main-test/java/accord/impl/list/ListResult.java b/accord-core/src/main-test/java/accord/impl/list/ListResult.java index adfa1fe3..c752bd0b 100644 --- a/accord-core/src/main-test/java/accord/impl/list/ListResult.java +++ b/accord-core/src/main-test/java/accord/impl/list/ListResult.java @@ -42,7 +42,7 @@ public class ListResult implements Result, Reply public final Keys responseKeys; public final int[][] read; // equal in size to keys.size() public final ListUpdate update; - private final Status status; + public final Status status; public ListResult(Status status, Id client, long requestId, TxnId txnId, Seekables<?, ?> readKeys, Keys responseKeys, int[][] read, ListUpdate update) { diff --git a/accord-core/src/main/java/accord/coordinate/PersistTxn.java b/accord-core/src/main/java/accord/coordinate/PersistTxn.java index c9139f01..797e8363 100644 --- a/accord-core/src/main/java/accord/coordinate/PersistTxn.java +++ b/accord-core/src/main/java/accord/coordinate/PersistTxn.java @@ -22,7 +22,6 @@ import accord.api.Result; import accord.local.Node; import accord.primitives.Deps; import accord.primitives.FullRoute; -import accord.primitives.Participants; import accord.primitives.Route; import accord.primitives.Timestamp; import accord.primitives.Txn; diff --git a/accord-core/src/main/java/accord/impl/DurabilityScheduling.java b/accord-core/src/main/java/accord/impl/DurabilityScheduling.java index ada18fc9..329ba009 100644 --- a/accord-core/src/main/java/accord/impl/DurabilityScheduling.java +++ b/accord-core/src/main/java/accord/impl/DurabilityScheduling.java @@ -314,10 +314,8 @@ public class DurabilityScheduling implements ConfigurationService.Listener index *= 2; numberOfSplits *= 2; } - if (fail instanceof Timeout) - logger.trace("{}: Exception coordinating ExclusiveSyncPoint for {} durability. Increased numberOfSplits to " + numberOfSplits, syncId, ranges, fail); - else - logger.debug("{}: Exception coordinating ExclusiveSyncPoint for {} durability. Increased numberOfSplits to " + numberOfSplits, syncId, ranges, fail); + + logger.trace("{}: Exception coordinating ExclusiveSyncPoint for {} durability. Increased numberOfSplits to " + numberOfSplits, syncId, ranges, fail); } } else @@ -339,10 +337,7 @@ public class DurabilityScheduling implements ConfigurationService.Listener .addCallback((success, fail) -> { if (fail != null && fail.getClass() != SyncPointErased.class) { - if (fail instanceof Timeout) - logger.trace("Exception coordinating shard durability for {}, will retry", exclusiveSyncPoint.route.toRanges(), fail); - else - logger.debug("Exception coordinating shard durability for {}, will retry", exclusiveSyncPoint.route.toRanges(), fail); + logger.trace("Exception coordinating shard durability for {}, will retry", exclusiveSyncPoint.route.toRanges(), fail); retryCoordinateDurability(node, exclusiveSyncPoint, nextIndex); } else diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java index cf64e6cd..a089036a 100644 --- a/accord-core/src/main/java/accord/local/Command.java +++ b/accord-core/src/main/java/accord/local/Command.java @@ -328,8 +328,8 @@ public abstract class Command implements CommonAttributes { default: throw new AssertionError("Unhandled Outcome: " + known.outcome); case Apply: - Invariants.checkState(writes != null, "Writes is null"); - Invariants.checkState(result != null, "Result is null"); + Invariants.checkState(writes != null, "Writes is null %s", validate); + Invariants.checkState(result != null, "Result is null %s", validate); break; case Invalidated: Invariants.checkState(validate.durability().isMaybeInvalidated(), "%s is not invalidated", validate.durability()); @@ -337,8 +337,8 @@ public abstract class Command implements CommonAttributes Invariants.checkState(validate.durability() != Local); case Erased: case WasApply: - Invariants.checkState(writes == null, "Writes exist"); - Invariants.checkState(result == null, "Results exist"); + Invariants.checkState(writes == null, "Writes exist for %s", validate); + Invariants.checkState(result == null, "Results exist %s", validate); break; } } diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java index e4e5a0a8..ca2ea321 100644 --- a/accord-core/src/main/java/accord/messages/CheckStatus.java +++ b/accord-core/src/main/java/accord/messages/CheckStatus.java @@ -18,6 +18,7 @@ package accord.messages; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import accord.api.Result; @@ -30,48 +31,43 @@ import accord.local.Node.Id; import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.primitives.SaveStatus; -import accord.primitives.Status; import accord.local.StoreParticipants; import accord.primitives.Ballot; +import accord.primitives.Known; import accord.primitives.KnownMap; -import accord.primitives.WithQuorum; import accord.primitives.PartialDeps; import accord.primitives.PartialTxn; import accord.primitives.Participants; import accord.primitives.ProgressToken; import accord.primitives.Ranges; import accord.primitives.Route; +import accord.primitives.SaveStatus; +import accord.primitives.Status; import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.primitives.Unseekables; +import accord.primitives.WithQuorum; import accord.primitives.Writes; import accord.topology.Topologies; import accord.utils.Invariants; import accord.utils.MapReduceConsume; - -import javax.annotation.Nonnull; +import accord.utils.async.Cancellable; import static accord.coordinate.Infer.InvalidIf.IfUncommitted; import static accord.coordinate.Infer.InvalidIf.IsNotInvalid; import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid; +import static accord.messages.TxnRequest.computeScope; +import static accord.primitives.Route.castToRoute; +import static accord.primitives.Route.isRoute; import static accord.primitives.Status.Durability; import static accord.primitives.Status.Durability.Local; import static accord.primitives.Status.Durability.Majority; import static accord.primitives.Status.Durability.ShardUniversal; import static accord.primitives.Status.Durability.Universal; - -import accord.primitives.Known; -import accord.utils.async.Cancellable; - -import static accord.primitives.Status.Invalidated; import static accord.primitives.Status.NotDefined; import static accord.primitives.Status.Stable; import static accord.primitives.Status.Truncated; -import static accord.messages.TxnRequest.computeScope; import static accord.primitives.WithQuorum.HasQuorum; -import static accord.primitives.Route.castToRoute; -import static accord.primitives.Route.isRoute; public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply> implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, CheckStatus.CheckStatusReply> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
