Migrate all size serialization to unsigned vint encoded Unsigned vint encoding is more efficient in both CPU and disk burden, so this patch employs it wherever sensible.
patch by benedict; reviewed by sylvain for CASSANDRA-10351 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/649a106c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/649a106c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/649a106c Branch: refs/heads/trunk Commit: 649a106c39b6a166c988fb647eaa33341e7371c4 Parents: 8134f09 Author: Benedict Elliott Smith <[email protected]> Authored: Wed Sep 16 10:33:22 2015 +0100 Committer: Benedict Elliott Smith <[email protected]> Committed: Wed Sep 16 13:16:47 2015 +0100 ---------------------------------------------------------------------- .../org/apache/cassandra/batchlog/Batch.java | 16 ++++----- src/java/org/apache/cassandra/db/Columns.java | 6 ++-- src/java/org/apache/cassandra/db/Mutation.java | 6 ++-- .../org/apache/cassandra/db/ReadCommand.java | 6 ++-- .../cassandra/db/SerializationHeader.java | 36 ++++++++++---------- src/java/org/apache/cassandra/db/Slices.java | 6 ++-- src/java/org/apache/cassandra/db/TypeSizes.java | 2 +- .../db/filter/ClusteringIndexNamesFilter.java | 6 ++-- .../cassandra/db/filter/ColumnFilter.java | 6 ++-- .../apache/cassandra/db/filter/DataLimits.java | 30 ++++++++-------- .../apache/cassandra/db/filter/RowFilter.java | 6 ++-- .../apache/cassandra/db/rows/EncodingStats.java | 18 +++++----- .../rows/UnfilteredRowIteratorSerializer.java | 6 ++-- .../cassandra/hints/EncodedHintMessage.java | 4 +-- src/java/org/apache/cassandra/hints/Hint.java | 8 ++--- .../org/apache/cassandra/hints/HintMessage.java | 6 ++-- .../cassandra/io/util/DataOutputPlus.java | 8 ++--- .../apache/cassandra/utils/ByteBufferUtil.java | 8 ++--- 18 files changed, 92 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/batchlog/Batch.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/batchlog/Batch.java b/src/java/org/apache/cassandra/batchlog/Batch.java index caa2682..e91e3ca 100644 --- a/src/java/org/apache/cassandra/batchlog/Batch.java +++ b/src/java/org/apache/cassandra/batchlog/Batch.java @@ -30,7 +30,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDSerializer; import static org.apache.cassandra.db.TypeSizes.sizeof; -import static org.apache.cassandra.db.TypeSizes.sizeofVInt; +import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public final class Batch { @@ -87,11 +87,11 @@ public final class Batch long size = UUIDSerializer.serializer.serializedSize(batch.id, version); size += sizeof(batch.creationTime); - size += sizeofVInt(batch.decodedMutations.size()); + size += sizeofUnsignedVInt(batch.decodedMutations.size()); for (Mutation mutation : batch.decodedMutations) { int mutationSize = (int) Mutation.serializer.serializedSize(mutation, version); - size += sizeofVInt(mutationSize); + size += sizeofUnsignedVInt(mutationSize); size += mutationSize; } @@ -105,10 +105,10 @@ public final class Batch UUIDSerializer.serializer.serialize(batch.id, out, version); out.writeLong(batch.creationTime); - out.writeVInt(batch.decodedMutations.size()); + out.writeUnsignedVInt(batch.decodedMutations.size()); for (Mutation mutation : batch.decodedMutations) { - out.writeVInt(Mutation.serializer.serializedSize(mutation, version)); + out.writeUnsignedVInt(Mutation.serializer.serializedSize(mutation, version)); Mutation.serializer.serialize(mutation, out, version); } } @@ -129,7 +129,7 @@ public final class Batch private static Collection<ByteBuffer> readEncodedMutations(DataInputPlus in) throws IOException { - int count = (int) in.readVInt(); + int count = (int) in.readUnsignedVInt(); ArrayList<ByteBuffer> mutations = new ArrayList<>(count); for (int i = 0; i < count; i++) @@ -140,12 +140,12 @@ public final class Batch private static Collection<Mutation> decodeMutations(DataInputPlus in, int version) throws IOException { - int count = (int) in.readVInt(); + int count = (int) in.readUnsignedVInt(); ArrayList<Mutation> mutations = new ArrayList<>(count); for (int i = 0; i < count; i++) { - in.readVInt(); // skip mutation size + in.readUnsignedVInt(); // skip mutation size mutations.add(Mutation.serializer.deserialize(in, version)); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/Columns.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java index 442f77f..cad295c 100644 --- a/src/java/org/apache/cassandra/db/Columns.java +++ b/src/java/org/apache/cassandra/db/Columns.java @@ -400,14 +400,14 @@ public class Columns extends AbstractCollection<ColumnDefinition> implements Col { public void serialize(Columns columns, DataOutputPlus out) throws IOException { - out.writeVInt(columns.size()); + out.writeUnsignedVInt(columns.size()); for (ColumnDefinition column : columns) ByteBufferUtil.writeWithVIntLength(column.name.bytes, out); } public long serializedSize(Columns columns) { - long size = TypeSizes.sizeofVInt(columns.size()); + long size = TypeSizes.sizeofUnsignedVInt(columns.size()); for (ColumnDefinition column : columns) size += ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes); return size; @@ -415,7 +415,7 @@ public class Columns extends AbstractCollection<ColumnDefinition> implements Col public Columns deserialize(DataInputPlus in, CFMetaData metadata) throws IOException { - int length = (int)in.readVInt(); + int length = (int)in.readUnsignedVInt(); BTree.Builder<ColumnDefinition> builder = BTree.builder(Comparator.naturalOrder()); builder.auto(false); for (int i = 0; i < length; i++) http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/Mutation.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index da7d13d..7696e04 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -279,7 +279,7 @@ public class Mutation implements IMutation } else { - out.writeVInt(size); + out.writeUnsignedVInt(size); } assert size > 0; @@ -301,7 +301,7 @@ public class Mutation implements IMutation } else { - size = (int)in.readVInt(); + size = (int)in.readUnsignedVInt(); } assert size > 0; @@ -343,7 +343,7 @@ public class Mutation implements IMutation } else { - size += TypeSizes.sizeofVInt(mutation.modifications.size()); + size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size()); } for (Map.Entry<UUID, PartitionUpdate> entry : mutation.modifications.entrySet()) http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/ReadCommand.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index e183963..d2e4bbc 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -564,7 +564,7 @@ public abstract class ReadCommand implements ReadQuery out.writeByte(command.kind.ordinal()); out.writeByte(digestFlag(command.isDigestQuery()) | thriftFlag(command.isForThrift()) | indexFlag(command.index.isPresent())); if (command.isDigestQuery()) - out.writeVInt(command.digestVersion()); + out.writeUnsignedVInt(command.digestVersion()); CFMetaData.serializer.serialize(command.metadata(), out, version); out.writeInt(command.nowInSec()); ColumnFilter.serializer.serialize(command.columnFilter(), out, version); @@ -586,7 +586,7 @@ public abstract class ReadCommand implements ReadQuery boolean isDigest = isDigest(flags); boolean isForThrift = isForThrift(flags); boolean hasIndex = hasIndex(flags); - int digestVersion = isDigest ? (int)in.readVInt() : 0; + int digestVersion = isDigest ? (int)in.readUnsignedVInt() : 0; CFMetaData metadata = CFMetaData.serializer.deserialize(in, version); int nowInSec = in.readInt(); ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata); @@ -623,7 +623,7 @@ public abstract class ReadCommand implements ReadQuery assert version >= MessagingService.VERSION_30; return 2 // kind + flags - + (command.isDigestQuery() ? TypeSizes.sizeofVInt(command.digestVersion()) : 0) + + (command.isDigestQuery() ? TypeSizes.sizeofUnsignedVInt(command.digestVersion()) : 0) + CFMetaData.serializer.serializedSize(command.metadata(), version) + TypeSizes.sizeof(command.nowInSec()) + ColumnFilter.serializer.serializedSize(command.columnFilter(), version) http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/SerializationHeader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java index 70c5401..decac49 100644 --- a/src/java/org/apache/cassandra/db/SerializationHeader.java +++ b/src/java/org/apache/cassandra/db/SerializationHeader.java @@ -164,17 +164,17 @@ public class SerializationHeader public void writeTimestamp(long timestamp, DataOutputPlus out) throws IOException { - out.writeVInt(timestamp - stats.minTimestamp); + out.writeUnsignedVInt(timestamp - stats.minTimestamp); } public void writeLocalDeletionTime(int localDeletionTime, DataOutputPlus out) throws IOException { - out.writeVInt(localDeletionTime - stats.minLocalDeletionTime); + out.writeUnsignedVInt(localDeletionTime - stats.minLocalDeletionTime); } public void writeTTL(int ttl, DataOutputPlus out) throws IOException { - out.writeVInt(ttl - stats.minTTL); + out.writeUnsignedVInt(ttl - stats.minTTL); } public void writeDeletionTime(DeletionTime dt, DataOutputPlus out) throws IOException @@ -185,17 +185,17 @@ public class SerializationHeader public long readTimestamp(DataInputPlus in) throws IOException { - return in.readVInt() + stats.minTimestamp; + return in.readUnsignedVInt() + stats.minTimestamp; } public int readLocalDeletionTime(DataInputPlus in) throws IOException { - return (int)in.readVInt() + stats.minLocalDeletionTime; + return (int)in.readUnsignedVInt() + stats.minLocalDeletionTime; } public int readTTL(DataInputPlus in) throws IOException { - return (int)in.readVInt() + stats.minTTL; + return (int)in.readUnsignedVInt() + stats.minTTL; } public DeletionTime readDeletionTime(DataInputPlus in) throws IOException @@ -207,17 +207,17 @@ public class SerializationHeader public long timestampSerializedSize(long timestamp) { - return TypeSizes.sizeofVInt(timestamp - stats.minTimestamp); + return TypeSizes.sizeofUnsignedVInt(timestamp - stats.minTimestamp); } public long localDeletionTimeSerializedSize(int localDeletionTime) { - return TypeSizes.sizeofVInt(localDeletionTime - stats.minLocalDeletionTime); + return TypeSizes.sizeofUnsignedVInt(localDeletionTime - stats.minLocalDeletionTime); } public long ttlSerializedSize(int ttl) { - return TypeSizes.sizeofVInt(ttl - stats.minTTL); + return TypeSizes.sizeofUnsignedVInt(ttl - stats.minTTL); } public long deletionTimeSerializedSize(DeletionTime dt) @@ -228,17 +228,17 @@ public class SerializationHeader public void skipTimestamp(DataInputPlus in) throws IOException { - in.readVInt(); + in.readUnsignedVInt(); } public void skipLocalDeletionTime(DataInputPlus in) throws IOException { - in.readVInt(); + in.readUnsignedVInt(); } public void skipTTL(DataInputPlus in) throws IOException { - in.readVInt(); + in.readUnsignedVInt(); } public void skipDeletionTime(DataInputPlus in) throws IOException @@ -418,7 +418,7 @@ public class SerializationHeader EncodingStats.serializer.serialize(header.stats, out); writeType(header.keyType, out); - out.writeVInt(header.clusteringTypes.size()); + out.writeUnsignedVInt(header.clusteringTypes.size()); for (AbstractType<?> type : header.clusteringTypes) writeType(type, out); @@ -432,7 +432,7 @@ public class SerializationHeader EncodingStats stats = EncodingStats.serializer.deserialize(in); AbstractType<?> keyType = readType(in); - int size = (int)in.readVInt(); + int size = (int)in.readUnsignedVInt(); List<AbstractType<?>> clusteringTypes = new ArrayList<>(size); for (int i = 0; i < size; i++) clusteringTypes.add(readType(in)); @@ -452,7 +452,7 @@ public class SerializationHeader int size = EncodingStats.serializer.serializedSize(header.stats); size += sizeofType(header.keyType); - size += TypeSizes.sizeofVInt(header.clusteringTypes.size()); + size += TypeSizes.sizeofUnsignedVInt(header.clusteringTypes.size()); for (AbstractType<?> type : header.clusteringTypes) size += sizeofType(type); @@ -463,7 +463,7 @@ public class SerializationHeader private void writeColumnsWithTypes(Map<ByteBuffer, AbstractType<?>> columns, DataOutputPlus out) throws IOException { - out.writeVInt(columns.size()); + out.writeUnsignedVInt(columns.size()); for (Map.Entry<ByteBuffer, AbstractType<?>> entry : columns.entrySet()) { ByteBufferUtil.writeWithVIntLength(entry.getKey(), out); @@ -473,7 +473,7 @@ public class SerializationHeader private long sizeofColumnsWithTypes(Map<ByteBuffer, AbstractType<?>> columns) { - long size = TypeSizes.sizeofVInt(columns.size()); + long size = TypeSizes.sizeofUnsignedVInt(columns.size()); for (Map.Entry<ByteBuffer, AbstractType<?>> entry : columns.entrySet()) { size += ByteBufferUtil.serializedSizeWithVIntLength(entry.getKey()); @@ -484,7 +484,7 @@ public class SerializationHeader private void readColumnsWithType(DataInputPlus in, Map<ByteBuffer, AbstractType<?>> typeMap) throws IOException { - int length = (int)in.readVInt(); + int length = (int)in.readUnsignedVInt(); for (int i = 0; i < length; i++) { ByteBuffer name = ByteBufferUtil.readWithVIntLength(in); http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/Slices.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java index db34c86..8fa9337 100644 --- a/src/java/org/apache/cassandra/db/Slices.java +++ b/src/java/org/apache/cassandra/db/Slices.java @@ -297,7 +297,7 @@ public abstract class Slices implements Iterable<Slice> public void serialize(Slices slices, DataOutputPlus out, int version) throws IOException { int size = slices.size(); - out.writeVInt(size); + out.writeUnsignedVInt(size); if (size == 0) return; @@ -312,7 +312,7 @@ public abstract class Slices implements Iterable<Slice> public long serializedSize(Slices slices, int version) { - long size = TypeSizes.sizeofVInt(slices.size()); + long size = TypeSizes.sizeofUnsignedVInt(slices.size()); if (slices.size() == 0) return size; @@ -329,7 +329,7 @@ public abstract class Slices implements Iterable<Slice> public Slices deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException { - int size = (int)in.readVInt(); + int size = (int)in.readUnsignedVInt(); if (size == 0) return NONE; http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/TypeSizes.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/TypeSizes.java b/src/java/org/apache/cassandra/db/TypeSizes.java index 7e5bd87..6543025 100644 --- a/src/java/org/apache/cassandra/db/TypeSizes.java +++ b/src/java/org/apache/cassandra/db/TypeSizes.java @@ -70,7 +70,7 @@ public final class TypeSizes public static int sizeofWithVIntLength(ByteBuffer value) { - return sizeofVInt(value.remaining()) + value.remaining(); + return sizeofUnsignedVInt(value.remaining()) + value.remaining(); } public static int sizeof(boolean value) http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java index e0bc533..d3a289a 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java @@ -240,7 +240,7 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter protected void serializeInternal(DataOutputPlus out, int version) throws IOException { ClusteringComparator comparator = (ClusteringComparator)clusterings.comparator(); - out.writeVInt(clusterings.size()); + out.writeUnsignedVInt(clusterings.size()); for (Clustering clustering : clusterings) Clustering.serializer.serialize(clustering, out, version, comparator.subtypes()); } @@ -248,7 +248,7 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter protected long serializedSizeInternal(int version) { ClusteringComparator comparator = (ClusteringComparator)clusterings.comparator(); - long size = TypeSizes.sizeofVInt(clusterings.size()); + long size = TypeSizes.sizeofUnsignedVInt(clusterings.size()); for (Clustering clustering : clusterings) size += Clustering.serializer.serializedSize(clustering, version, comparator.subtypes()); return size; @@ -260,7 +260,7 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter { ClusteringComparator comparator = metadata.comparator; BTreeSet.Builder<Clustering> clusterings = BTreeSet.builder(comparator); - int size = (int)in.readVInt(); + int size = (int)in.readUnsignedVInt(); for (int i = 0; i < size; i++) clusterings.add(Clustering.serializer.deserialize(in, version, comparator.subtypes())); http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/filter/ColumnFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java index 29b3164..1a4573e 100644 --- a/src/java/org/apache/cassandra/db/filter/ColumnFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnFilter.java @@ -368,7 +368,7 @@ public class ColumnFilter if (selection.subSelections != null) { - out.writeVInt(selection.subSelections.size()); + out.writeUnsignedVInt(selection.subSelections.size()); for (ColumnSubselection subSel : selection.subSelections.values()) ColumnSubselection.serializer.serialize(subSel, out, version); } @@ -393,7 +393,7 @@ public class ColumnFilter if (hasSubSelections) { subSelections = TreeMultimap.create(Comparator.<ColumnIdentifier>naturalOrder(), Comparator.<ColumnSubselection>naturalOrder()); - int size = (int)in.readVInt(); + int size = (int)in.readUnsignedVInt(); for (int i = 0; i < size; i++) { ColumnSubselection subSel = ColumnSubselection.serializer.deserialize(in, version, metadata); @@ -417,7 +417,7 @@ public class ColumnFilter if (selection.subSelections != null) { - size += TypeSizes.sizeofVInt(selection.subSelections.size()); + size += TypeSizes.sizeofUnsignedVInt(selection.subSelections.size()); for (ColumnSubselection subSel : selection.subSelections.values()) size += ColumnSubselection.serializer.serializedSize(subSel, version); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/filter/DataLimits.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java index 0d6f816..75c8290 100644 --- a/src/java/org/apache/cassandra/db/filter/DataLimits.java +++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java @@ -644,21 +644,21 @@ public abstract class DataLimits case CQL_LIMIT: case CQL_PAGING_LIMIT: CQLLimits cqlLimits = (CQLLimits)limits; - out.writeVInt(cqlLimits.rowLimit); - out.writeVInt(cqlLimits.perPartitionLimit); + out.writeUnsignedVInt(cqlLimits.rowLimit); + out.writeUnsignedVInt(cqlLimits.perPartitionLimit); out.writeBoolean(cqlLimits.isDistinct); if (limits.kind() == Kind.CQL_PAGING_LIMIT) { CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits; ByteBufferUtil.writeWithVIntLength(pagingLimits.lastReturnedKey, out); - out.writeVInt(pagingLimits.lastReturnedKeyRemaining); + out.writeUnsignedVInt(pagingLimits.lastReturnedKeyRemaining); } break; case THRIFT_LIMIT: case SUPER_COLUMN_COUNTING_LIMIT: ThriftLimits thriftLimits = (ThriftLimits)limits; - out.writeVInt(thriftLimits.partitionLimit); - out.writeVInt(thriftLimits.cellPerPartitionLimit); + out.writeUnsignedVInt(thriftLimits.partitionLimit); + out.writeUnsignedVInt(thriftLimits.cellPerPartitionLimit); break; } } @@ -670,19 +670,19 @@ public abstract class DataLimits { case CQL_LIMIT: case CQL_PAGING_LIMIT: - int rowLimit = (int)in.readVInt(); - int perPartitionLimit = (int)in.readVInt(); + int rowLimit = (int)in.readUnsignedVInt(); + int perPartitionLimit = (int)in.readUnsignedVInt(); boolean isDistinct = in.readBoolean(); if (kind == Kind.CQL_LIMIT) return new CQLLimits(rowLimit, perPartitionLimit, isDistinct); ByteBuffer lastKey = ByteBufferUtil.readWithVIntLength(in); - int lastRemaining = (int)in.readVInt(); + int lastRemaining = (int)in.readUnsignedVInt(); return new CQLPagingLimits(rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining); case THRIFT_LIMIT: case SUPER_COLUMN_COUNTING_LIMIT: - int partitionLimit = (int)in.readVInt(); - int cellPerPartitionLimit = (int)in.readVInt(); + int partitionLimit = (int)in.readUnsignedVInt(); + int cellPerPartitionLimit = (int)in.readUnsignedVInt(); return kind == Kind.THRIFT_LIMIT ? new ThriftLimits(partitionLimit, cellPerPartitionLimit) : new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit); @@ -698,21 +698,21 @@ public abstract class DataLimits case CQL_LIMIT: case CQL_PAGING_LIMIT: CQLLimits cqlLimits = (CQLLimits)limits; - size += TypeSizes.sizeofVInt(cqlLimits.rowLimit); - size += TypeSizes.sizeofVInt(cqlLimits.perPartitionLimit); + size += TypeSizes.sizeofUnsignedVInt(cqlLimits.rowLimit); + size += TypeSizes.sizeofUnsignedVInt(cqlLimits.perPartitionLimit); size += TypeSizes.sizeof(cqlLimits.isDistinct); if (limits.kind() == Kind.CQL_PAGING_LIMIT) { CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits; size += ByteBufferUtil.serializedSizeWithVIntLength(pagingLimits.lastReturnedKey); - size += TypeSizes.sizeofVInt(pagingLimits.lastReturnedKeyRemaining); + size += TypeSizes.sizeofUnsignedVInt(pagingLimits.lastReturnedKeyRemaining); } break; case THRIFT_LIMIT: case SUPER_COLUMN_COUNTING_LIMIT: ThriftLimits thriftLimits = (ThriftLimits)limits; - size += TypeSizes.sizeofVInt(thriftLimits.partitionLimit); - size += TypeSizes.sizeofVInt(thriftLimits.cellPerPartitionLimit); + size += TypeSizes.sizeofUnsignedVInt(thriftLimits.partitionLimit); + size += TypeSizes.sizeofUnsignedVInt(thriftLimits.cellPerPartitionLimit); break; default: throw new AssertionError(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/filter/RowFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java index bbec004..bf92efb 100644 --- a/src/java/org/apache/cassandra/db/filter/RowFilter.java +++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java @@ -748,7 +748,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException { out.writeBoolean(filter instanceof ThriftFilter); - out.writeVInt(filter.expressions.size()); + out.writeUnsignedVInt(filter.expressions.size()); for (Expression expr : filter.expressions) Expression.serializer.serialize(expr, out, version); } @@ -756,7 +756,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> public RowFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException { boolean forThrift = in.readBoolean(); - int size = (int)in.readVInt(); + int size = (int)in.readUnsignedVInt(); List<Expression> expressions = new ArrayList<>(size); for (int i = 0; i < size; i++) expressions.add(Expression.serializer.deserialize(in, version, metadata)); @@ -768,7 +768,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> public long serializedSize(RowFilter filter, int version) { long size = 1 // forThrift - + TypeSizes.sizeofVInt(filter.expressions.size()); + + TypeSizes.sizeofUnsignedVInt(filter.expressions.size()); for (Expression expr : filter.expressions) size += Expression.serializer.serializedSize(expr, version); return size; http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/rows/EncodingStats.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/rows/EncodingStats.java b/src/java/org/apache/cassandra/db/rows/EncodingStats.java index 2970fa3..955ffc7 100644 --- a/src/java/org/apache/cassandra/db/rows/EncodingStats.java +++ b/src/java/org/apache/cassandra/db/rows/EncodingStats.java @@ -230,23 +230,23 @@ public class EncodingStats { public void serialize(EncodingStats stats, DataOutputPlus out) throws IOException { - out.writeVInt(stats.minTimestamp - TIMESTAMP_EPOCH); - out.writeVInt(stats.minLocalDeletionTime - DELETION_TIME_EPOCH); - out.writeVInt(stats.minTTL - TTL_EPOCH); + out.writeUnsignedVInt(stats.minTimestamp - TIMESTAMP_EPOCH); + out.writeUnsignedVInt(stats.minLocalDeletionTime - DELETION_TIME_EPOCH); + out.writeUnsignedVInt(stats.minTTL - TTL_EPOCH); } public int serializedSize(EncodingStats stats) { - return TypeSizes.sizeofVInt(stats.minTimestamp - TIMESTAMP_EPOCH) - + TypeSizes.sizeofVInt(stats.minLocalDeletionTime - DELETION_TIME_EPOCH) - + TypeSizes.sizeofVInt(stats.minTTL - TTL_EPOCH); + return TypeSizes.sizeofUnsignedVInt(stats.minTimestamp - TIMESTAMP_EPOCH) + + TypeSizes.sizeofUnsignedVInt(stats.minLocalDeletionTime - DELETION_TIME_EPOCH) + + TypeSizes.sizeofUnsignedVInt(stats.minTTL - TTL_EPOCH); } public EncodingStats deserialize(DataInputPlus in) throws IOException { - long minTimestamp = in.readVInt() + TIMESTAMP_EPOCH; - int minLocalDeletionTime = (int)in.readVInt() + DELETION_TIME_EPOCH; - int minTTL = (int)in.readVInt() + TTL_EPOCH; + long minTimestamp = in.readUnsignedVInt() + TIMESTAMP_EPOCH; + int minLocalDeletionTime = (int)in.readUnsignedVInt() + DELETION_TIME_EPOCH; + int minTTL = (int)in.readUnsignedVInt() + TTL_EPOCH; return new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java index e1b2c09..df006d7 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java @@ -123,7 +123,7 @@ public class UnfilteredRowIteratorSerializer UnfilteredSerializer.serializer.serialize(staticRow, header, out, version); if (rowEstimate >= 0) - out.writeVInt(rowEstimate); + out.writeUnsignedVInt(rowEstimate); while (iterator.hasNext()) UnfilteredSerializer.serializer.serialize(iterator.next(), header, out, version); @@ -159,7 +159,7 @@ public class UnfilteredRowIteratorSerializer size += UnfilteredSerializer.serializer.serializedSize(staticRow, header, version); if (rowEstimate >= 0) - size += TypeSizes.sizeofVInt(rowEstimate); + size += TypeSizes.sizeofUnsignedVInt(rowEstimate); while (iterator.hasNext()) size += UnfilteredSerializer.serializer.serializedSize(iterator.next(), header, version); @@ -191,7 +191,7 @@ public class UnfilteredRowIteratorSerializer if (hasStatic) staticRow = UnfilteredSerializer.serializer.deserializeStaticRow(in, header, new SerializationHelper(metadata, version, flag)); - int rowEstimate = hasRowEstimate ? (int)in.readVInt() : -1; + int rowEstimate = hasRowEstimate ? (int)in.readUnsignedVInt() : -1; return new Header(header, key, isReversed, false, partitionDeletion, staticRow, rowEstimate); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/hints/EncodedHintMessage.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java index 56727fc..4fe05ac 100644 --- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java +++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java @@ -66,7 +66,7 @@ final class EncodedHintMessage throw new IllegalArgumentException("serializedSize() called with non-matching version " + version); long size = UUIDSerializer.serializer.serializedSize(message.hostId, version); - size += TypeSizes.sizeofVInt(message.hint.remaining()); + size += TypeSizes.sizeofUnsignedVInt(message.hint.remaining()); size += message.hint.remaining(); return size; } @@ -77,7 +77,7 @@ final class EncodedHintMessage throw new IllegalArgumentException("serialize() called with non-matching version " + version); UUIDSerializer.serializer.serialize(message.hostId, out, version); - out.writeVInt(message.hint.remaining()); + out.writeUnsignedVInt(message.hint.remaining()); out.write(message.hint); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/hints/Hint.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java index c88c494..cbb5e74 100644 --- a/src/java/org/apache/cassandra/hints/Hint.java +++ b/src/java/org/apache/cassandra/hints/Hint.java @@ -27,7 +27,7 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import static org.apache.cassandra.db.TypeSizes.sizeof; -import static org.apache.cassandra.db.TypeSizes.sizeofVInt; +import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; /** * Encapsulates the hinted mutation, its creation time, and the gc grace seconds param for each table involved. @@ -111,7 +111,7 @@ public final class Hint public long serializedSize(Hint hint, int version) { long size = sizeof(hint.creationTime); - size += sizeofVInt(hint.gcgs); + size += sizeofUnsignedVInt(hint.gcgs); size += Mutation.serializer.serializedSize(hint.mutation, version); return size; } @@ -119,14 +119,14 @@ public final class Hint public void serialize(Hint hint, DataOutputPlus out, int version) throws IOException { out.writeLong(hint.creationTime); - out.writeVInt(hint.gcgs); + out.writeUnsignedVInt(hint.gcgs); Mutation.serializer.serialize(hint.mutation, out, version); } public Hint deserialize(DataInputPlus in, int version) throws IOException { long creationTime = in.readLong(); - int gcgs = (int) in.readVInt(); + int gcgs = (int) in.readUnsignedVInt(); return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/hints/HintMessage.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/HintMessage.java b/src/java/org/apache/cassandra/hints/HintMessage.java index 6296a8c..e78738d 100644 --- a/src/java/org/apache/cassandra/hints/HintMessage.java +++ b/src/java/org/apache/cassandra/hints/HintMessage.java @@ -86,7 +86,7 @@ public final class HintMessage long size = UUIDSerializer.serializer.serializedSize(message.hostId, version); long hintSize = Hint.serializer.serializedSize(message.hint, version); - size += TypeSizes.sizeofVInt(hintSize); + size += TypeSizes.sizeofUnsignedVInt(hintSize); size += hintSize; return size; @@ -102,7 +102,7 @@ public final class HintMessage * We are serializing the hint size so that the receiver of the message could gracefully handle * deserialize failure when a table had been dropped, by simply skipping the unread bytes. */ - out.writeVInt(Hint.serializer.serializedSize(message.hint, version)); + out.writeUnsignedVInt(Hint.serializer.serializedSize(message.hint, version)); Hint.serializer.serialize(message.hint, out, version); } @@ -116,7 +116,7 @@ public final class HintMessage { UUID hostId = UUIDSerializer.serializer.deserialize(in, version); - long hintSize = in.readVInt(); + long hintSize = in.readUnsignedVInt(); BytesReadTracker countingIn = new BytesReadTracker(in); try { http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/io/util/DataOutputPlus.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java index 60a5727..a9dbb68 100644 --- a/src/java/org/apache/cassandra/io/util/DataOutputPlus.java +++ b/src/java/org/apache/cassandra/io/util/DataOutputPlus.java @@ -49,11 +49,11 @@ public interface DataOutputPlus extends DataOutput } /** - * Think hard before opting for an unsigned encoding. Is this going to bite someone because some day - * they might need to pass in a sentinel value using negative numbers? Is the risk worth it - * to save a few bytes? + * This is more efficient for storing unsigned values, both in storage and CPU burden. * - * Signed, not a fan of unsigned values in protocols and formats + * Note that it is still possible to store negative values, they just take up more space. + * So this method doesn't forbid e.g. negative sentinel values in future, if they need to be snuck in. + * A protocol version bump can then be introduced to improve efficiency. */ default void writeUnsignedVInt(long i) throws IOException { http://git-wip-us.apache.org/repos/asf/cassandra/blob/649a106c/src/java/org/apache/cassandra/utils/ByteBufferUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index a05c3c8..70d4bd5 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -293,7 +293,7 @@ public class ByteBufferUtil public static void writeWithVIntLength(ByteBuffer bytes, DataOutputPlus out) throws IOException { - out.writeVInt(bytes.remaining()); + out.writeUnsignedVInt(bytes.remaining()); out.write(bytes); } @@ -332,7 +332,7 @@ public class ByteBufferUtil public static ByteBuffer readWithVIntLength(DataInputPlus in) throws IOException { - int length = (int)in.readVInt(); + int length = (int)in.readUnsignedVInt(); if (length < 0) throw new IOException("Corrupt (negative) value length encountered"); @@ -348,12 +348,12 @@ public class ByteBufferUtil public static int serializedSizeWithVIntLength(ByteBuffer buffer) { int size = buffer.remaining(); - return TypeSizes.sizeofVInt(size) + size; + return TypeSizes.sizeofUnsignedVInt(size) + size; } public static void skipWithVIntLength(DataInputPlus in) throws IOException { - int length = (int)in.readVInt(); + int length = (int)in.readUnsignedVInt(); if (length < 0) throw new IOException("Corrupt (negative) value length encountered");
