This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit a8270c786bf75e0c77f084d45f9900c3c111f97f Author: Alex Petrov <[email protected]> AuthorDate: Fri Sep 18 15:50:40 2020 +0300 Patch introduces the following changes: 1. Add “progressive” generators in tests (i.e., ones that start with simple schemas/data types, such as ones that use longs and progress to more complex ones, such as strings, doubles, and other data types, including reverse ones). 2. Fix support for reverse types in clustering columns 3. Remove adjustEntropyDomain and improve Float and Double data types, making them byte-ordered and switching them to use less entropy (3 bytes for float and 7 bytes for double). Patch by Alex Petrov, for CASSANDRA-15348 --- .../src/harry/corruptor/AddExtraRowCorruptor.java | 20 ++- harry-core/src/harry/ddl/ColumnSpec.java | 72 ++-------- harry-core/src/harry/ddl/SchemaGenerators.java | 120 +++++++++++++++- harry-core/src/harry/ddl/SchemaSpec.java | 1 + harry-core/src/harry/generators/Bijections.java | 112 ++++++++++----- .../src/harry/generators/DataGenerators.java | 154 +++++++++------------ .../src/harry/generators/StringBijection.java | 1 + harry-core/src/harry/generators/Surjections.java | 9 ++ harry-core/src/harry/model/DataTracker.java | 2 +- harry-core/src/harry/model/OpSelectors.java | 10 +- harry-core/src/harry/model/QuiescentChecker.java | 16 ++- harry-core/src/harry/reconciler/Reconciler.java | 19 ++- harry-core/src/harry/runner/Query.java | 5 +- harry-core/src/harry/runner/QuerySelector.java | 41 ++++-- harry-core/src/harry/util/TestRunner.java | 2 +- .../test/harry/generators/DataGeneratorsTest.java | 84 +++++++---- harry-core/test/harry/model/OpSelectorsTest.java | 16 ++- harry-core/test/harry/op/RowVisitorTest.java | 57 ++++---- 18 files changed, 465 insertions(+), 276 deletions(-) diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java index 85ae502..9a73867 100644 --- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java +++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java @@ -21,6 +21,9 @@ package harry.corruptor; import java.util.HashSet; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import harry.data.ResultSetRow; import harry.ddl.SchemaSpec; import harry.model.OpSelectors; @@ -31,6 +34,8 @@ import harry.runner.Query; public class AddExtraRowCorruptor implements QueryResponseCorruptor { + private static final Logger logger = LoggerFactory.getLogger(AddExtraRowCorruptor.class); + private final SchemaSpec schema; private final OpSelectors.MonotonicClock clock; private final OpSelectors.DescriptorSelector descriptorSelector; @@ -57,18 +62,20 @@ public class AddExtraRowCorruptor implements QueryResponseCorruptor maxLts = Math.max(maxLts, row.lts[i]); } - if (cds.size() >= descriptorSelector.maxPartitionSize()) - return false; + boolean partitionIsFull = cds.size() >= descriptorSelector.maxPartitionSize(); - long cd; long attempt = 0; - do + long cd = descriptorSelector.randomCd(query.pd, attempt, schema);; + while (!query.match(cd) || cds.contains(cd)) { - cd = descriptorSelector.randomCd(query.pd, attempt, schema); + if (partitionIsFull) + // We can't pick from the existing CDs, so let's try to come up with a new one that would match the query + cd += descriptorSelector.randomCd(query.pd, attempt, schema); + else + cd = descriptorSelector.randomCd(query.pd, attempt, schema); if (attempt++ == 1000) return false; } - while (!query.match(cd) || cds.contains(cd)); long[] vds = descriptorSelector.vds(query.pd, cd, maxLts, 0, schema); @@ -76,6 +83,7 @@ public class AddExtraRowCorruptor implements QueryResponseCorruptor // still won't help since we can't use it anyways, since collisions between a // written value and tombstone are resolved in favour of tombstone, so we're // just going to take the next lts. + logger.info("Corrupting the resultset by writing a row with cd {}", cd); sut.execute(WriteHelper.inflateInsert(schema, query.pd, cd, vds, clock.rts(maxLts) + 1)); return true; } diff --git a/harry-core/src/harry/ddl/ColumnSpec.java b/harry-core/src/harry/ddl/ColumnSpec.java index 623a6e7..ee4a774 100644 --- a/harry-core/src/harry/ddl/ColumnSpec.java +++ b/harry-core/src/harry/ddl/ColumnSpec.java @@ -104,11 +104,6 @@ public class ColumnSpec<T> return type.generator().inflate(current); } - public long adjustEntropyDomain(long current) - { - return type.generator().adjustEntropyDomain(current); - } - public long deflate(T value) { return type.generator().deflate(value); @@ -162,7 +157,10 @@ public class ColumnSpec<T> public abstract Bijections.Bijection<T> generator(); - public abstract int maxSize(); + public int maxSize() + { + return generator().byteSize(); + } public String toString() { @@ -176,84 +174,56 @@ public class ColumnSpec<T> { return Bijections.INT8_GENERATOR; } - - public int maxSize() - { - return Byte.BYTES; - } }; + public static final DataType<Short> int16Type = new DataType<Short>("smallint") { public Bijections.Bijection<Short> generator() { return Bijections.INT16_GENERATOR; } - - public int maxSize() - { - return Short.BYTES; - } }; + public static final DataType<Integer> int32Type = new DataType<Integer>("int") { public Bijections.Bijection<Integer> generator() { return Bijections.INT32_GENERATOR; } - - public int maxSize() - { - return Integer.BYTES; - } }; + public static final DataType<Long> int64Type = new DataType<Long>("bigint") { public Bijections.Bijection<Long> generator() { return Bijections.INT64_GENERATOR; } - - public int maxSize() - { - return Long.BYTES; - } }; + public static final DataType<Boolean> booleanType = new DataType<Boolean>("boolean") { public Bijections.Bijection<Boolean> generator() { return Bijections.BOOLEAN_GENERATOR; } - - public int maxSize() - { - return Byte.BYTES; - } }; + public static final DataType<Float> floatType = new DataType<Float>("float") { public Bijections.Bijection<Float> generator() { return Bijections.FLOAT_GENERATOR; } - - public int maxSize() - { - return Float.BYTES; - } }; + public static final DataType<Double> doubleType = new DataType<Double>("double") { public Bijections.Bijection<Double> generator() { return Bijections.DOUBLE_GENERATOR; } - - public int maxSize() - { - return Double.BYTES; - } }; + public static final DataType<String> asciiType = new DataType<String>("ascii") { private final Bijections.Bijection<String> gen = new StringBijection(); @@ -262,11 +232,6 @@ public class ColumnSpec<T> { return gen; } - - public int maxSize() - { - return Long.BYTES; - } }; public static DataType<String> asciiType(int nibbleSize, int maxRandomNibbles) @@ -279,11 +244,6 @@ public class ColumnSpec<T> { return gen; } - - public int maxSize() - { - return Long.BYTES; - } }; } @@ -293,11 +253,6 @@ public class ColumnSpec<T> { return Bijections.UUID_GENERATOR; } - - public int maxSize() - { - return Long.BYTES; - } }; public static final DataType<Date> timestampType = new DataType<Date>("timestamp") @@ -306,11 +261,6 @@ public class ColumnSpec<T> { return Bijections.TIMESTAMP_GENERATOR; } - - public int maxSize() - { - return Long.BYTES; - } }; public static final Collection<DataType<?>> DATA_TYPES = ImmutableList.of( diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java b/harry-core/src/harry/ddl/SchemaGenerators.java index fd46bb8..aecc189 100644 --- a/harry-core/src/harry/ddl/SchemaGenerators.java +++ b/harry-core/src/harry/ddl/SchemaGenerators.java @@ -169,9 +169,14 @@ public class SchemaGenerators public Builder partitionKeySpec(int minCols, int maxCols, ColumnSpec.DataType<?>... columnTypes) { + return partitionKeySpec(minCols, maxCols, Arrays.asList(columnTypes)); + } + + public Builder partitionKeySpec(int minCols, int maxCols, Collection<ColumnSpec.DataType<?>> columnTypes) + { this.minPks = minCols; this.maxPks = maxCols; - this.pkGenerator = columnSpecGenerator(Arrays.asList(columnTypes), "pk", ColumnSpec.Kind.PARTITION_KEY); + this.pkGenerator = columnSpecGenerator(columnTypes, "pk", ColumnSpec.Kind.PARTITION_KEY); return this; } @@ -189,9 +194,14 @@ public class SchemaGenerators public Builder clusteringKeySpec(int minCols, int maxCols, ColumnSpec.DataType<?>... columnTypes) { + return clusteringKeySpec(minCols, maxCols, Arrays.asList(columnTypes)); + } + + public Builder clusteringKeySpec(int minCols, int maxCols, Collection<ColumnSpec.DataType<?>> columnTypes) + { this.minCks = minCols; this.maxCks = maxCols; - this.ckGenerator = columnSpecGenerator(Arrays.asList(columnTypes), "ck", ColumnSpec.Kind.CLUSTERING); + this.ckGenerator = columnSpecGenerator(columnTypes, "ck", ColumnSpec.Kind.CLUSTERING); return this; } @@ -209,9 +219,14 @@ public class SchemaGenerators public Builder regularColumnSpec(int minCols, int maxCols, ColumnSpec.DataType<?>... columnTypes) { + return this.regularColumnSpec(minCols, maxCols, Arrays.asList(columnTypes)); + } + + public Builder regularColumnSpec(int minCols, int maxCols, Collection<ColumnSpec.DataType<?>> columnTypes) + { this.minRegular = minCols; this.maxRegular = maxCols; - this.regularGenerator = columnSpecGenerator(Arrays.asList(columnTypes), "regular", ColumnSpec.Kind.REGULAR); + this.regularGenerator = columnSpecGenerator(columnTypes, "regular", ColumnSpec.Kind.REGULAR); return this; } @@ -292,4 +307,101 @@ public class SchemaGenerators ColumnSpec.asciiType(5, 10)) .surjection(); } -} + + public static final String DEFAULT_KEYSPACE_NAME = "harry"; + private static final String DEFAULT_PREFIX = "table_"; + private static final AtomicInteger counter = new AtomicInteger(); + private static final Supplier<String> tableNameSupplier = () -> DEFAULT_PREFIX + counter.getAndIncrement(); + + // simplest schema gen, nothing can go wrong with it + public static final Surjections.Surjection<SchemaSpec> longOnlySpecBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(1, 1, ColumnSpec.int64Type) + .clusteringKeySpec(1, 1, ColumnSpec.int64Type) + .regularColumnSpec(1, 10, ColumnSpec.int64Type) + .surjection(); + + private static final ColumnSpec.DataType<String> simpleStringType = ColumnSpec.asciiType(4, 10); + private static final Surjections.Surjection<SchemaSpec> longAndStringSpecBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(2, 2, ColumnSpec.int64Type, simpleStringType) + .clusteringKeySpec(2, 2, ColumnSpec.int64Type, simpleStringType) + .regularColumnSpec(1, 10, ColumnSpec.int64Type, simpleStringType) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec> longOnlyWithReverseSpecBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(1, 1, ColumnSpec.int64Type) + .clusteringKeySpec(1, 1, ColumnSpec.ReversedType.getInstance(ColumnSpec.int64Type)) + .regularColumnSpec(1, 10, ColumnSpec.int64Type) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec> longAndStringSpecWithReversedLongBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(2, 2, ColumnSpec.int64Type, simpleStringType) + .clusteringKeySpec(2, 2, ColumnSpec.ReversedType.getInstance(ColumnSpec.int64Type), simpleStringType) + .regularColumnSpec(1, 10, ColumnSpec.int64Type, simpleStringType) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec> longAndStringSpecWithReversedStringBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(2, 2, ColumnSpec.int64Type, simpleStringType) + .clusteringKeySpec(2, 2, ColumnSpec.int64Type, ColumnSpec.ReversedType.getInstance(simpleStringType)) + .regularColumnSpec(1, 10, ColumnSpec.int64Type, simpleStringType) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec> longAndStringSpecWithReversedBothBuilder = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(2, 2, ColumnSpec.int64Type, simpleStringType) + .clusteringKeySpec(2, 2, ColumnSpec.ReversedType.getInstance(ColumnSpec.int64Type), ColumnSpec.ReversedType.getInstance(simpleStringType)) + .regularColumnSpec(1, 10, ColumnSpec.int64Type, simpleStringType) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec> withAllFeaturesEnabled = new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, tableNameSupplier) + .partitionKeySpec(1, 4, columnTypes) + .clusteringKeySpec(1, 4, clusteringKeyTypes) + .regularColumnSpec(1, 10, columnTypes) + .surjection(); + + public static final Surjections.Surjection<SchemaSpec>[] PROGRESSIVE_GENERATORS = new Surjections.Surjection[]{ + longOnlySpecBuilder, + longAndStringSpecBuilder, + longOnlyWithReverseSpecBuilder, + longAndStringSpecWithReversedLongBuilder, + longAndStringSpecWithReversedStringBuilder, + longAndStringSpecWithReversedBothBuilder, + withAllFeaturesEnabled + }; + // Create schema generators that would produce tables starting with just a few features, progressing to use more + public static Supplier<SchemaSpec> progression(int switchAfter) + { + Supplier<SchemaSpec>[] generators = new Supplier[PROGRESSIVE_GENERATORS.length]; + for (int i = 0; i < generators.length; i++) + generators[i] = PROGRESSIVE_GENERATORS[i].toSupplier(); + + return new Supplier<SchemaSpec>() + { + private final AtomicInteger counter = new AtomicInteger(); + public SchemaSpec get() + { + int idx = (counter.getAndIncrement() / switchAfter) % generators.length; + SchemaSpec spec = generators[idx].get(); + int tries = 100; + while ((spec.ckGenerator.byteSize() != Long.BYTES || spec.pkGenerator.byteSize() != Long.BYTES) && tries > 0) + { + System.out.println("Skipping schema, since it doesn't have enough entropy bits available: " + spec.compile().cql()); + spec = generators[idx].get(); + tries--; + } + + assert tries > 0 : String.format("Max number of tries exceeded on generator %d, can't generate a needed schema", idx); + return spec; + } + + + }; + } + + public static int DEFAULT_SWITCH_AFTER = 5; + public static int GENERATORS_COUNT = PROGRESSIVE_GENERATORS.length; + public static int DEFAULT_RUNS = DEFAULT_SWITCH_AFTER * PROGRESSIVE_GENERATORS.length; + + public static Supplier<SchemaSpec> progression() + { + return progression(DEFAULT_SWITCH_AFTER); // would generate 30 tables before wrapping around + } +} \ No newline at end of file diff --git a/harry-core/src/harry/ddl/SchemaSpec.java b/harry-core/src/harry/ddl/SchemaSpec.java index 52c4767..336bd67 100644 --- a/harry-core/src/harry/ddl/SchemaSpec.java +++ b/harry-core/src/harry/ddl/SchemaSpec.java @@ -32,6 +32,7 @@ import harry.operations.Relation; import harry.util.BitSet; // TODO: improve API of this class +// TODO: forbid schemas where pk and cks don't add up to 64 bits (for now) public class SchemaSpec { public interface SchemaSpecFactory diff --git a/harry-core/src/harry/generators/Bijections.java b/harry-core/src/harry/generators/Bijections.java index 43d78c7..d0d6532 100644 --- a/harry-core/src/harry/generators/Bijections.java +++ b/harry-core/src/harry/generators/Bijections.java @@ -49,6 +49,7 @@ public class Bijections long deflate(T value); + // TODO: byteSize is great, but you know what's better? Bit size! For example, for `boolean`, we only need a single bit. int byteSize(); int compare(long l, long r); @@ -57,6 +58,41 @@ public class Bijections { return descriptor & Bytes.bytePatternFor(byteSize()); } + + default long minValue() + { + return minForSize(byteSize()); + } + + default long maxValue() + { + return maxForSize(byteSize()); + } + + default boolean byteOrdered() + { + return false; + } + } + + protected static long minForSize(int size) + { + long min = 1L << (size * Byte.SIZE - 1); + + if (size < Long.BYTES) + min ^= Bytes.signMaskFor(size); + + return min; + } + + protected static long maxForSize(int size) + { + long max = Bytes.bytePatternFor(size) >>> 1; + + if (size < Long.BYTES) + max ^= Bytes.signMaskFor(size); + + return max; } // TODO: two points: @@ -64,6 +100,11 @@ public class Bijections // * since these data types are quite specialized, we do not strictly need complex interface for them, it might // be easier to even create a special type for these. We need randomness source in cases of more complex generation, // but not really here. + + /** + * Reverse type is different from the regular one in that will generate values + * that will sort the order that is opposite to the order of descriptor. + */ public static class ReverseBijection<T> implements Bijection<T> { private final Bijection<T> delegate; @@ -75,12 +116,12 @@ public class Bijections public T inflate(long descriptor) { - return delegate.inflate(descriptor * -1); + return delegate.inflate(descriptor * -1 - 1); } public long deflate(T value) { - return -1 * delegate.deflate(value); + return -1 * (delegate.deflate(value) + 1); } public int byteSize() @@ -92,12 +133,6 @@ public class Bijections { return delegate.compare(r, l); } - - public long adjustEntropyDomain(long descriptor) - { - long pattern = Bytes.BYTES[byteSize() - 1]; - return descriptor & (pattern >> 1); - } } public static class LongGenerator implements Bijection<Long> @@ -219,7 +254,6 @@ public class Bijections return Byte.compare((byte) l, (byte) r); } - // TODO: this this right? Why +1? public long adjustEntropyDomain(long descriptor) { return (descriptor & 1) + 1; @@ -228,6 +262,8 @@ public class Bijections public static class FloatGenerator implements Bijection<Float> { + private static final int SIZE = Float.BYTES - 1; + public Float inflate(long current) { return inflatePrimitive(current); @@ -235,31 +271,28 @@ public class Bijections protected float inflatePrimitive(long current) { - long tmp = current & 0xffffffffL; - tmp ^= ((tmp >> 31) & 0x7fffffffL); - return Float.intBitsToFloat((int) tmp); + return Float.intBitsToFloat((int) current); } public long deflate(Float value) { - int tmp = Float.floatToRawIntBits(value); - tmp ^= ((tmp >> 31) & 0x7fffffffL); - return tmp; + return Float.floatToRawIntBits(value); } - public int compare(long l, long r) + // In other words, there's no way we can extend entropy to a sign + public boolean byteOrdered() { - return Float.compare(inflatePrimitive(l), inflatePrimitive(r)); + return true; } - public int byteSize() + public int compare(long l, long r) { - return Float.BYTES; + return Float.compare(inflatePrimitive(l), inflatePrimitive(r)); } - public long adjustEntropyDomain(long descriptor) + public int byteSize() { - return (~0x8F000000L & descriptor) & 0xffffffffL; + return SIZE; } } @@ -267,17 +300,24 @@ public class Bijections { public float inflatePrimitive(long current) { - return super.inflatePrimitive(current) * -1; + return super.inflatePrimitive(current - 1) * -1; } public long deflate(Float value) { - return super.deflate(value * -1); + return super.deflate(value * -1 ) + 1; + } + + public int compare(long l, long r) + { + return super.compare(r, l); } } public static class DoubleGenerator implements Bijection<Double> { + private static int SIZE = Double.BYTES - 1; + public Double inflate(long current) { return inflatePrimitive(current); @@ -285,15 +325,12 @@ public class Bijections protected double inflatePrimitive(long current) { - current = current ^ ((current >> 63) & 0x7fffffffffffffffL); return Double.longBitsToDouble(current); } public long deflate(Double value) { - long current = Double.doubleToRawLongBits(value); - current = current ^ ((current >> 63) & 0x7fffffffffffffffL); - return current; + return Double.doubleToRawLongBits(value); } public int compare(long l, long r) @@ -303,26 +340,30 @@ public class Bijections public int byteSize() { - return Double.BYTES; + return SIZE; } - public long adjustEntropyDomain(long descriptor) + public boolean byteOrdered() { - return (~0x8F00000000000000L & descriptor); + return true; } } - public static class ReverseDoubleGenerator extends DoubleGenerator { public double inflatePrimitive(long current) { - return super.inflatePrimitive(current) * -1; + return super.inflatePrimitive(current - 1) * -1; } public long deflate(Double value) { - return super.deflate(value * -1); + return super.deflate(value * -1) + 1; + } + + public int compare(long l, long r) + { + return super.compare(r, l); } } @@ -371,10 +412,5 @@ public class Bijections { return Long.BYTES; } - - public long adjustEntropyDomain(long descriptor) - { - return descriptor & (Bytes.bytePatternFor(byteSize() >> 1)); - } } } \ No newline at end of file diff --git a/harry-core/src/harry/generators/DataGenerators.java b/harry-core/src/harry/generators/DataGenerators.java index 9acd957..9d3593b 100644 --- a/harry-core/src/harry/generators/DataGenerators.java +++ b/harry-core/src/harry/generators/DataGenerators.java @@ -190,12 +190,12 @@ public class DataGenerators return 0; } - protected long minValueInternal(int idx) + public long minValue(int idx) { return 0; } - protected long maxValueInternal(int idx) + public long maxValue(int idx) { return 0; } @@ -244,51 +244,19 @@ public class DataGenerators public long minValue() { - return columns.get(0).isReversed() ? maxForSize(byteSize()) : minForSize(byteSize()); + return Bijections.minForSize(byteSize()); } public long maxValue() { - return columns.get(0).isReversed() ? minForSize(byteSize()) : maxForSize(byteSize()); - } - - protected static long minForSize(int size) - { - long min = 1L << (size * Byte.SIZE - 1); - - if (size < Long.BYTES) - min ^= Bytes.signMaskFor(size); - - return min; - } - - protected long maxForSize(int size) - { - long max = Bytes.bytePatternFor(size) >>> 1; - - if (size < Long.BYTES) - max ^= Bytes.signMaskFor(size); - - return max; + return Bijections.maxForSize(byteSize()); } /** * Min value for a segment: 0, possibly with an inverted 0 sign for stitching. - * Similar thing can be achieved by */ - public long minValue(int idx) - { - return columns.get(idx).isReversed() ? maxValueInternal(idx) : minValueInternal(idx); - } - - public long maxValue(int idx) - { - return columns.get(idx).isReversed() ? minValueInternal(idx) : maxValueInternal(idx); - } - - protected abstract long minValueInternal(int idx); - - protected abstract long maxValueInternal(int idx); + public abstract long minValue(int idx); + public abstract long maxValue(int idx); } static class SinglePartKeyGenerator extends KeyGenerator @@ -306,37 +274,50 @@ public class DataGenerators public long[] slice(long descriptor) { - long adjusted = adjustEntropyDomain(descriptor); - long[] res = new long[]{ adjusted }; - assert adjusted == stitch(res); - return res; + if (shouldInvertSign()) + descriptor ^= Bytes.signMaskFor(byteSize()); + + descriptor = adjustEntropyDomain(descriptor); + return new long[]{ descriptor }; } public long stitch(long[] parts) { - return parts[0]; + long descriptor = parts[0]; + + if (shouldInvertSign()) + descriptor ^= Bytes.signMaskFor(byteSize()); + + return adjustEntropyDomain(descriptor); } - public long minValueInternal(int idx) + public long minValue(int idx) { - return minForSize(totalSize); + assert idx == 0; + return keyGen.minValue(); } - public long maxValueInternal(int idx) + public long maxValue(int idx) { - return maxForSize(totalSize); + assert idx == 0; + return keyGen.maxValue(); } public Object[] inflate(long descriptor) { - return new Object[]{ keyGen.inflate(descriptor) }; + long[] sliced = slice(descriptor); + return new Object[]{ keyGen.inflate(sliced[0]) }; + } + + public boolean shouldInvertSign() + { + return totalSize != Long.BYTES && !keyGen.byteOrdered(); } public long deflate(Object[] value) { long descriptor = keyGen.deflate(value[0]); - descriptor &= Bytes.bytePatternFor(totalSize); - return descriptor; + return stitch(new long[] { descriptor }); } public int byteSize() @@ -348,13 +329,6 @@ public class DataGenerators { return Long.compare(l, r); } - - public long adjustEntropyDomain(long descriptor) - { - descriptor &= (Bytes.bytePatternFor(totalSize) >> 1); - descriptor = keyGen.adjustEntropyDomain(descriptor); - return descriptor; - } } public static class MultiPartKeyGenerator extends KeyGenerator @@ -386,32 +360,41 @@ public class DataGenerators return DataGenerators.inflateKey(columns, descriptor, slice(descriptor)); } - public long adjustEntropyDomain(long descriptor) - { - // We can't simply trim the value here, mostly because of the values like - // long and double that can change the value during normalization in addition - // to trimming it. - return stitch(slice(descriptor)); - } - // Checks whether we need to invert a slice sign to preserve order of the sliced descriptor public boolean shouldInvertSign(int idx) { - int maxSliceSize = columns.get(idx).generator().byteSize(); + Bijections.Bijection<?> gen = columns.get(idx).generator(); + + int maxSliceSize = gen.byteSize(); int actualSliceSize = sizes[idx]; if (idx == 0) { - // Signed representation of the first value would follow the sorting of the - // long value itself, which means that we have invert sign of the first piece if: - // * not all entropy bytes are consumed - // * we do not have enough entropy bytes to set the sign bit of the value - // TODO: I think maxPieceSize != pieceSize is not right here; it should be <= ??? - return totalSize != Long.BYTES || maxSliceSize != actualSliceSize; + // We consume a sign of a descriptor (long, long), (int, int), etc. + if (totalSize == Long.BYTES) + { + // if we use only 3 bytes for a 4-byte int, or 4 bytes for a 8-byte int, + // they're effectively unsigned/byte-ordered, so their order won't match + if (maxSliceSize > actualSliceSize) + return true; + // Since descriptor is signed, invert sign of all byte-ordered types, since + // their order won't match + else + return gen.byteOrdered(); + } + // We do not consume a sign of a descriptor (float, tinyint), (int, tinyint), etc, + // so we have to only invert signs of the values, since their order doesn't match. + else + { + assert maxSliceSize == actualSliceSize; + return !gen.byteOrdered(); + } } - - // We invert sign of all subsequent chunks if their signs match - return maxSliceSize == actualSliceSize; + else if (gen.byteOrdered()) + return false; + else + // We invert sign of all subsequent chunks if they have enough entropy to have a sign bit set + return maxSliceSize == actualSliceSize; } public long[] slice(long descriptor) @@ -423,12 +406,10 @@ public class DataGenerators final int size = sizes[i]; long piece = descriptor >> ((pos - size) * Byte.SIZE); - piece &= Bytes.bytePatternFor(size); - if (shouldInvertSign(i)) piece ^= Bytes.signMaskFor(size); - piece = columns.get(i).adjustEntropyDomain(piece); + piece &= Bytes.bytePatternFor(size); pieces[i] = piece; pos -= size; @@ -455,21 +436,22 @@ public class DataGenerators return stitched; } - protected long minValueInternal(int idx) + + public long minValue(int idx) { - int size = sizes[idx]; - long res = 0; + long res = columns.get(idx).generator().minValue(); + // Inverting sign is important for range queries and RTs, since we're + // making boundaries that'll be stitched later. if (shouldInvertSign(idx)) - res ^= Bytes.signMaskFor(size); + res ^= Bytes.signMaskFor(sizes[idx]); return res; } - protected long maxValueInternal(int idx) + public long maxValue(int idx) { - int size = sizes[idx]; - long res = Bytes.bytePatternFor(size); + long res = columns.get(idx).generator().maxValue(); if (shouldInvertSign(idx)) - res ^= Bytes.signMaskFor(size); + res ^= Bytes.signMaskFor(sizes[idx]); return res; } diff --git a/harry-core/src/harry/generators/StringBijection.java b/harry-core/src/harry/generators/StringBijection.java index 7a7b8c2..46b58e7 100644 --- a/harry-core/src/harry/generators/StringBijection.java +++ b/harry-core/src/harry/generators/StringBijection.java @@ -79,6 +79,7 @@ public class StringBijection implements Bijections.Bijection<String> public static int getByte(long l, int idx) { int b = (int) ((l >> (Long.BYTES - idx - 1) * Byte.SIZE) & 0xff); + // TODO: make use of `byteOrdered` to avoid flipping this bit if (idx == 0) b ^= 0x80; return b; diff --git a/harry-core/src/harry/generators/Surjections.java b/harry-core/src/harry/generators/Surjections.java index 1caf71d..78a2b3b 100644 --- a/harry-core/src/harry/generators/Surjections.java +++ b/harry-core/src/harry/generators/Surjections.java @@ -26,6 +26,8 @@ import java.util.function.Function; import java.util.function.LongFunction; import java.util.function.Supplier; +import com.google.common.annotations.VisibleForTesting; + public class Surjections { public static <T> Surjection<T> constant(Supplier<T> constant) @@ -154,5 +156,12 @@ public class Surjections } }; } + + @VisibleForTesting + default Supplier<T> toSupplier() + { + RandomGenerator rng = new PcgRSUFast(0, 0); + return () -> inflate(rng.next()); + } } } diff --git a/harry-core/src/harry/model/DataTracker.java b/harry-core/src/harry/model/DataTracker.java index 78815d5..dfd2503 100644 --- a/harry-core/src/harry/model/DataTracker.java +++ b/harry-core/src/harry/model/DataTracker.java @@ -105,7 +105,7 @@ public class DataTracker } @VisibleForTesting - void forceLts(long maxSeen, long maxComplete) + public void forceLts(long maxSeen, long maxComplete) { this.maxSeenLts.set(maxSeen); this.maxCompleteLts.set(maxComplete); diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java index d86d436..ad8e046 100644 --- a/harry-core/src/harry/model/OpSelectors.java +++ b/harry-core/src/harry/model/OpSelectors.java @@ -25,7 +25,9 @@ import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; import harry.core.Configuration; +import harry.ddl.ColumnSpec; import harry.ddl.SchemaSpec; +import harry.generators.Bytes; import harry.generators.PCGFastPure; import harry.generators.RngUtils; import harry.generators.Surjections; @@ -185,8 +187,9 @@ public interface OpSelectors { if (mask.isSet(col)) { - long vd = vd(pd, cd, lts, opId, col); - vds[col] = schema.regularColumns.get(col).adjustEntropyDomain(vd); + ColumnSpec spec = schema.regularColumns.get(col); + long vd = vd(pd, cd, lts, opId, col) & Bytes.signMaskFor(spec.type.maxSize()); + vds[col] = vd; } else { @@ -278,7 +281,8 @@ public interface OpSelectors public long minLtsFor(long pd) { - return minLtsAt(rng.sequenceNumber(pd, PARTITION_DESCRIPTOR_STREAM_ID)); + long sequenceNumber = rng.sequenceNumber(pd, PARTITION_DESCRIPTOR_STREAM_ID); + return minLtsAt(sequenceNumber); } public long positionFor(long lts) diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java index 94a4b37..f7b38a3 100644 --- a/harry-core/src/harry/model/QuiescentChecker.java +++ b/harry-core/src/harry/model/QuiescentChecker.java @@ -19,7 +19,9 @@ package harry.model; import java.util.Arrays; +import java.util.Collection; import java.util.Iterator; +import java.util.List; import com.google.common.annotations.VisibleForTesting; @@ -67,8 +69,10 @@ public class QuiescentChecker implements Model assert maxCompeteLts == maxSeenLts : "Runner hasn't settled down yet. " + "Quiescent model can't be reliably used in such cases."; - Iterator<ResultSetRow> actual = SelectHelper.execute(sut, clock, query).iterator(); - Iterator<Reconciler.RowState> expected = reconciler.inflatePartitionState(query.pd, maxSeenLts, query).iterator(query.reverse); + List<ResultSetRow> actualRows = SelectHelper.execute(sut, clock, query); + Iterator<ResultSetRow> actual = actualRows.iterator(); + Collection<Reconciler.RowState> expectedRows = reconciler.inflatePartitionState(query.pd, maxSeenLts, query).rows(query.reverse); + Iterator<Reconciler.RowState> expected = expectedRows.iterator(); while (actual.hasNext() && expected.hasNext()) { @@ -92,8 +96,12 @@ public class QuiescentChecker implements Model if (actual.hasNext() || expected.hasNext()) { - throw new ValidationException("Expected results to have the same number of results, but %s result iterator has more results", - actual.hasNext() ? "actual" : "expected"); + throw new ValidationException("Expected results to have the same number of results, but %s result iterator has more results." + + "\nExpected: %s" + + "\nActual: %s", + actual.hasNext() ? "actual" : "expected", + expectedRows, + actualRows); } } diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java index 061cec3..50b03c6 100644 --- a/harry-core/src/harry/reconciler/Reconciler.java +++ b/harry-core/src/harry/reconciler/Reconciler.java @@ -20,6 +20,7 @@ package harry.reconciler; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.NavigableMap; @@ -62,7 +63,7 @@ public class Reconciler this.querySelector = querySelector; } - public PartitionState inflatePartitionState(long pd, long maxLts, Query query) + public PartitionState inflatePartitionState(final long pd, long maxLts, Query query) { List<Ranges.Range> ranges = new ArrayList<>(); // TODO: we should think of a single-pass algorithm that would allow us to inflate all deletes and range deletes for a partition @@ -82,6 +83,7 @@ public class Reconciler }; long currentLts = pdSelector.minLtsFor(pd); + while (currentLts <= maxLts && currentLts >= 0) { partitionVisitor.visitPartition(currentLts); @@ -96,10 +98,13 @@ public class Reconciler { public void operation(long lts, long pd, long cd, long m, long opId) { - if (!query.match(cd) || rts.isShadowed(cd, lts)) + OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId); + + if (opType == OpSelectors.OperationKind.DELETE_ROW || opType == OpSelectors.OperationKind.DELETE_RANGE) return; - OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId); + if (!query.match(cd) || rts.isShadowed(cd, lts)) + return; if (opType == OpSelectors.OperationKind.WRITE) { @@ -208,6 +213,14 @@ public class Reconciler return rows.values().iterator(); } + + public Collection<RowState> rows(boolean reverse) + { + if (reverse) + return rows.descendingMap().values(); + + return rows.values(); + } } public static class RowState diff --git a/harry-core/src/harry/runner/Query.java b/harry-core/src/harry/runner/Query.java index 91ba80b..d0742c0 100644 --- a/harry-core/src/harry/runner/Query.java +++ b/harry-core/src/harry/runner/Query.java @@ -37,7 +37,6 @@ import harry.operations.Relation; import harry.util.Ranges; import static harry.model.ExhaustiveChecker.FORWARD_COMPARATOR; -import static harry.model.ExhaustiveChecker.REVERSE_COMPARATOR; public abstract class Query { @@ -192,7 +191,7 @@ public abstract class Query SchemaSpec schemaSpec) { super(kind, pd, reverse, allRelations, schemaSpec); - assert cdMin <= cdMax : String.format("%d should be <= %d\nAll relations: %s.\nReverse: %s", cdMin, cdMax, allRelations, reverse); + assert cdMin != cdMax || (minRelation.isInclusive() && maxRelation.isInclusive()); this.cdMin = cdMin; this.cdMax = cdMax; this.minRelation = minRelation; @@ -206,7 +205,7 @@ public abstract class Query public boolean match(long cd) { - ExhaustiveChecker.LongComparator cmp = schemaSpec.clusteringKeys.get(0).isReversed() ? REVERSE_COMPARATOR : FORWARD_COMPARATOR; + ExhaustiveChecker.LongComparator cmp = FORWARD_COMPARATOR; boolean res = minRelation.match(cmp, cd, cdMin) && maxRelation.match(cmp, cd, cdMax); if (!logger.isDebugEnabled()) return res; diff --git a/harry-core/src/harry/runner/QuerySelector.java b/harry-core/src/harry/runner/QuerySelector.java index 2aafb8a..00a567a 100644 --- a/harry-core/src/harry/runner/QuerySelector.java +++ b/harry-core/src/harry/runner/QuerySelector.java @@ -132,6 +132,7 @@ public class QuerySelector { long v = sliced[i]; DataGenerators.KeyGenerator gen = schema.ckGenerator; + ColumnSpec column = schema.clusteringKeys.get(i); int idx = i; LongSupplier maxSupplier = () -> gen.maxValue(idx); LongSupplier minSupplier = () -> gen.minValue(idx); @@ -145,8 +146,17 @@ public class QuerySelector else if (i == nonEqFrom) { relations.add(Relation.relation(relationKind(isGt, isEquals), schema.clusteringKeys.get(i), v)); - minBound[i] = isGt ? v : minSupplier.getAsLong(); - maxBound[i] = isGt ? maxSupplier.getAsLong() : v; + + if (column.isReversed()) + { + minBound[i] = isGt ? minSupplier.getAsLong() : v; + maxBound[i] = isGt ? v : maxSupplier.getAsLong(); + } + else + { + minBound[i] = isGt ? v : minSupplier.getAsLong(); + maxBound[i] = isGt ? maxSupplier.getAsLong() : v; + } } else { @@ -155,14 +165,16 @@ public class QuerySelector minBound[i] = minSupplier.getAsLong(); maxBound[i] = maxSupplier.getAsLong(); } + else if (i > 0 && schema.clusteringKeys.get(i - 1).isReversed()) + maxBound[i] = minBound[i] = isGt ? minSupplier.getAsLong() : maxSupplier.getAsLong(); else - { - minBound[i] = !isGt ? minSupplier.getAsLong() : maxSupplier.getAsLong(); - maxBound[i] = isGt ? maxSupplier.getAsLong() : minSupplier.getAsLong(); - } + maxBound[i] = minBound[i] = isGt ? maxSupplier.getAsLong() : minSupplier.getAsLong(); } } + if (schema.clusteringKeys.get(nonEqFrom).isReversed()) + isGt = !isGt; + min = schema.ckGenerator.stitch(minBound); max = schema.ckGenerator.stitch(maxBound); @@ -229,14 +241,23 @@ public class QuerySelector long maxLocked = Math.max(minBound[lock], maxBound[lock]); relations.add(Relation.relation(relationKind(true, isMinEq), col, minLocked)); - minBound[i] = minLocked; + + minBound[i] = col.isReversed() ? maxLocked : minLocked; relations.add(Relation.relation(relationKind(false, isMaxEq), col, maxLocked)); - maxBound[i] = maxLocked; + maxBound[i] = col.isReversed() ? minLocked : maxLocked; } else { - minBound[i] = isMinEq ? schema.ckGenerator.minValue(i) : schema.ckGenerator.maxValue(i); - maxBound[i] = isMaxEq ? schema.ckGenerator.maxValue(i) : schema.ckGenerator.minValue(i); + if (i > 0 && schema.clusteringKeys.get(i - 1).isReversed()) + { + minBound[i] = isMinEq ? schema.ckGenerator.maxValue(i) : schema.ckGenerator.minValue(i); + maxBound[i] = isMaxEq ? schema.ckGenerator.minValue(i) : schema.ckGenerator.maxValue(i); + } + else + { + minBound[i] = isMinEq ? schema.ckGenerator.minValue(i) : schema.ckGenerator.maxValue(i); + maxBound[i] = isMaxEq ? schema.ckGenerator.maxValue(i) : schema.ckGenerator.minValue(i); + } } } diff --git a/harry-core/src/harry/util/TestRunner.java b/harry-core/src/harry/util/TestRunner.java index 908d6f2..9abea62 100644 --- a/harry-core/src/harry/util/TestRunner.java +++ b/harry-core/src/harry/util/TestRunner.java @@ -29,7 +29,7 @@ public class TestRunner { private static final int CYCLES = 100; - private static final RandomGenerator rand = RandomGenerator.forTests(6371747244598697093L); + protected static final RandomGenerator rand = RandomGenerator.forTests(6371747244598697093L); public static <T1, T2> void test(Generator<T1> gen1, Generator<T2> gen2, diff --git a/harry-core/test/harry/generators/DataGeneratorsTest.java b/harry-core/test/harry/generators/DataGeneratorsTest.java index a3f6492..386644f 100644 --- a/harry-core/test/harry/generators/DataGeneratorsTest.java +++ b/harry-core/test/harry/generators/DataGeneratorsTest.java @@ -25,11 +25,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; -import java.util.Random; import java.util.function.BiConsumer; import java.util.function.Consumer; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import harry.ddl.ColumnSpec; @@ -38,7 +38,7 @@ import static harry.generators.Bijections.Bijection; public class DataGeneratorsTest { - private static final int RUNS = 500; + private static final int RUNS = 100; private static final RandomGenerator rand = RandomGenerator.forTests(1); @Test @@ -74,7 +74,7 @@ public class DataGeneratorsTest ColumnSpec.int32Type, ColumnSpec.int64Type); testRequiredBytes(sizes(4, 4), ColumnSpec.int32Type, ColumnSpec.int32Type); - testRequiredBytes(sizes(4, 4), + testRequiredBytes(sizes(4, 3), ColumnSpec.int32Type, ColumnSpec.floatType); testRequiredBytes(sizes(4, 4), ColumnSpec.int64Type, ColumnSpec.int64Type); @@ -162,36 +162,40 @@ public class DataGeneratorsTest { for (int i = 1; i < 5; i++) { - Iterator<ColumnSpec.DataType[]> iter = permutations(i, - ColumnSpec.DataType.class, - ColumnSpec.int8Type, - ColumnSpec.asciiType, - ColumnSpec.int16Type, - ColumnSpec.int32Type, - ColumnSpec.int64Type, - ColumnSpec.floatType, - ColumnSpec.doubleType - ); - while (iter.hasNext()) + for (boolean asReversed : new boolean[]{ false, true }) { - ColumnSpec.DataType[] types = iter.next(); - try + Iterator<ColumnSpec.DataType[]> iter = permutations(i, + ColumnSpec.DataType.class, + ColumnSpec.int8Type, + ColumnSpec.asciiType, + ColumnSpec.int16Type, + ColumnSpec.int32Type, + ColumnSpec.int64Type, + ColumnSpec.floatType, + ColumnSpec.doubleType + ); + + while (iter.hasNext()) { - testKeyGenerators(true, types); - testKeyGenerators( false, types); - } - catch (Throwable t) - { - throw new AssertionError("Caught error for the type combination " + Arrays.toString(types) , t); + ColumnSpec.DataType[] types = iter.next(); + try + { + testKeyGenerators(asReversed, types); + } + catch (Throwable t) + { + throw new AssertionError("Caught error for the type combination " + Arrays.toString(types), t); + } } } } } - @Test + @Ignore + @Test // this one is mostly useful when the above test fails and you need a quicker turnaround public void testSomeKeyGenerators() { - Iterator<ColumnSpec.DataType[]> iter = Collections.singletonList(new ColumnSpec.DataType[]{ ColumnSpec.asciiType, ColumnSpec.int64Type }).iterator(); + Iterator<ColumnSpec.DataType[]> iter = Collections.singletonList(new ColumnSpec.DataType[]{ ColumnSpec.int32Type, ColumnSpec.int32Type }).iterator(); while (iter.hasNext()) { @@ -222,7 +226,14 @@ public class DataGeneratorsTest // test some edge cases testKeyGenerators(0, 0, keyGenerator); testKeyGenerators(0xffffffffffffffffL, 0xffffffffffffffffL, keyGenerator); + testKeyGenerators(keyGenerator.minValue(), keyGenerator.maxValue(), keyGenerator); + testKeyGenerators(0, keyGenerator.minValue(), keyGenerator); + testKeyGenerators(0, keyGenerator.maxValue(), keyGenerator); long descriptor = rand.next(); + testKeyGenerators(descriptor, 0, keyGenerator); + testKeyGenerators(descriptor, keyGenerator.minValue(), keyGenerator); + testKeyGenerators(descriptor, keyGenerator.maxValue(), keyGenerator); + testKeyGenerators(descriptor, 0xffffffffffffffffL, keyGenerator); testKeyGenerators(descriptor, descriptor + 1, keyGenerator); testKeyGenerators(descriptor, descriptor - 1, keyGenerator); testKeyGenerators(descriptor, descriptor, keyGenerator); @@ -254,10 +265,13 @@ public class DataGeneratorsTest keyGenerator.deflate(value1)); assertDescriptorsEqual(descriptor2, keyGenerator.deflate(value2)); - Assert.assertEquals(String.format("%s (%s) and %s (%s) sort wrong", + + Assert.assertEquals(String.format("%s %s %s and %s %s %s have different order. ", Arrays.toString(value1), - Long.toHexString(descriptor1), + toSignString(compare(value1, value2, keyGenerator.columns)), Arrays.toString(value2), + Long.toHexString(descriptor1), + toSignString(Long.compare(descriptor1, descriptor2)), Long.toHexString(descriptor2)), normalize(Long.compare(descriptor1, descriptor2)), compare(value1, value2, keyGenerator.columns)); @@ -293,6 +307,8 @@ public class DataGeneratorsTest { testInverse(Bijections.INT64_GENERATOR); testOrderPreserving(Bijections.INT64_GENERATOR); + testInverse(new Bijections.ReverseBijection(Bijections.INT64_GENERATOR)); + testOrderPreserving(new Bijections.ReverseBijection(Bijections.INT64_GENERATOR), true); } @Test @@ -332,6 +348,11 @@ public class DataGeneratorsTest public static <T extends Comparable> void testOrderPreserving(Bijection<T> gen) { + testOrderPreserving(gen, false); + } + + public static <T extends Comparable> void testOrderPreserving(Bijection<T> gen, boolean reverse) + { test(gen, gen, (v1, v2) -> { long v1Descriptor = gen.adjustEntropyDomain(v1.descriptor); @@ -341,7 +362,7 @@ public class DataGeneratorsTest Long.toHexString(v1Descriptor), v2.value, Long.toHexString(v2Descriptor)), - normalize(Long.compare(v1Descriptor, v2Descriptor)), + normalize(Long.compare(v1Descriptor, v2Descriptor)) * (reverse ? -1 : 1), normalize(v1.value.compareTo(v2.value))); }); } @@ -385,6 +406,14 @@ public class DataGeneratorsTest } + public static String toSignString(int l) + { + if (l == 0) + return "="; + else if (l > 0) + return ">"; + return "<"; + } public static int normalize(int l) { if (l == 0) @@ -402,6 +431,7 @@ public class DataGeneratorsTest { Comparable comparableA = (Comparable) a[i]; Comparable comparableB = (Comparable) b[i]; + int cmp = comparableA.compareTo(comparableB); if (cmp != 0) { diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java index 560dd93..6c55a7f 100644 --- a/harry-core/test/harry/model/OpSelectorsTest.java +++ b/harry-core/test/harry/model/OpSelectorsTest.java @@ -27,12 +27,16 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.function.BiConsumer; +import java.util.function.Supplier; import org.junit.Assert; import org.junit.Test; import harry.ddl.ColumnSpec; +import harry.ddl.SchemaGenerators; import harry.ddl.SchemaSpec; +import harry.generators.PcgRSUFast; +import harry.generators.RandomGenerator; import harry.generators.Surjections; import harry.generators.distribution.Distribution; import harry.model.sut.NoOpSut; @@ -157,11 +161,13 @@ public class OpSelectorsTest @Test public void ckSelectorTest() { - SchemaSpec schema = new SchemaSpec("ks", "tbl1", - Collections.singletonList(ColumnSpec.pk("pk1", ColumnSpec.asciiType)), - Collections.singletonList(ColumnSpec.ck("ck1", ColumnSpec.asciiType)), - Collections.singletonList(ColumnSpec.regularColumn("v1", ColumnSpec.asciiType))); + Supplier<SchemaSpec> gen = SchemaGenerators.progression(5); + for (int i = 0; i < 30; i++) + ckSelectorTest(gen.get()); + } + public void ckSelectorTest(SchemaSpec schema) + { OpSelectors.Rng rng = new OpSelectors.PCGFast(1); OpSelectors.PdSelector pdSelector = new OpSelectors.DefaultPdSelector(rng, 10, 10); OpSelectors.DescriptorSelector ckSelector = new OpSelectors.DefaultDescriptorSelector(rng, @@ -218,7 +224,9 @@ public class OpSelectorsTest }).get(); for (int lts = 0; lts < 1000; lts++) + { partitionVisitor.visitPartition(lts); + } for (Collection<Long> value : partitionMap.values()) Assert.assertEquals(10, value.size()); diff --git a/harry-core/test/harry/op/RowVisitorTest.java b/harry-core/test/harry/op/RowVisitorTest.java index e071b40..c2db8ed 100644 --- a/harry-core/test/harry/op/RowVisitorTest.java +++ b/harry-core/test/harry/op/RowVisitorTest.java @@ -19,14 +19,20 @@ package harry.op; import java.util.function.Consumer; +import java.util.function.Supplier; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import harry.ddl.SchemaGenerators; import harry.ddl.SchemaSpec; import harry.generators.Generator; +import harry.generators.PcgRSUFast; +import harry.generators.RandomGenerator; +import harry.generators.Surjections; import harry.generators.distribution.Distribution; +import harry.model.OpSelectorsTest; import harry.runner.DefaultRowVisitor; import harry.model.clock.OffsetClock; import harry.model.OpSelectors; @@ -40,14 +46,17 @@ import static harry.model.OpSelectors.DefaultDescriptorSelector.DEFAULT_OP_TYPE_ public class RowVisitorTest extends CQLTester { + @Before + public void beforeTest() throws Throwable { + super.beforeTest(); + schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", SchemaGenerators.DEFAULT_KEYSPACE_NAME)); + } + @Test public void rowWriteGeneratorTest() { - Generator<SchemaSpec> specGenerator = SchemaGenerators.schema(KEYSPACE) - .partitionKeyColumnCount(1, 5) - .clusteringColumnCount(1, 5) - .regularColumnCount(2, 5) - .generator(); + Supplier<SchemaSpec> specGenerator = SchemaGenerators.progression(5); + RandomGenerator rand = RandomGenerator.forTests(6371747244598697093L); OpSelectors.Rng opRng = new OpSelectors.PCGFast(1); OpSelectors.DefaultDescriptorSelector descriptorSelector = new OpSelectors.DefaultDescriptorSelector(new OpSelectors.PCGFast(1L), @@ -60,29 +69,27 @@ public class RowVisitorTest extends CQLTester new Distribution.ScaledDistribution(2, 30), 100); - test(specGenerator, - (schema) -> { - createTable(schema.compile().cql()); - return (rng) -> { - DefaultRowVisitor visitor = new DefaultRowVisitor(schema, - new OffsetClock(10000), - descriptorSelector, - new QuerySelector(schema, - new OpSelectors.DefaultPdSelector(opRng, 10, 10), - descriptorSelector, - opRng)); - long[] descriptors = rng.next(4); + for (int i = 0; i < SchemaGenerators.PROGRESSIVE_GENERATORS.length * 5; i++) + { + SchemaSpec schema = specGenerator.get(); + createTable(schema.compile().cql()); + + DefaultRowVisitor visitor = new DefaultRowVisitor(schema, + new OffsetClock(10000), + descriptorSelector, + new QuerySelector(schema, + new OpSelectors.DefaultPdSelector(opRng, 10, 10), + descriptorSelector, + opRng)); + long[] descriptors = rand.next(4); - return visitor.write(Math.abs(descriptors[0]), - descriptors[1], - descriptors[2], - descriptors[3]); - }; - }, - (Consumer<CompiledStatement>) this::execute); + execute(visitor.write(Math.abs(descriptors[0]), + descriptors[1], + descriptors[2], + descriptors[3])); + } } - public void execute(CompiledStatement statement) { try --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
