This is an automated email from the ASF dual-hosted git repository. bdeggleston pushed a commit to branch cassandra-3.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push: new 53f604d Validate value sizes in LegacyLayout 53f604d is described below commit 53f604dc1789a800dbcbc3c8aee77f8f36b8b5db Author: Blake Eggleston <bdeggles...@gmail.com> AuthorDate: Wed Oct 23 15:40:07 2019 -0700 Validate value sizes in LegacyLayout Patch by Blake Eggleston; Reviewed by Benedict Elliot Smith and Sam Tunnicliffe for CASSANDRA-15373 --- CHANGES.txt | 1 + src/java/org/apache/cassandra/db/LegacyLayout.java | 21 ++ .../apache/cassandra/db/marshal/AbstractType.java | 8 + .../org/apache/cassandra/db/LegacyLayoutTest.java | 1 - .../cassandra/db/LegacyLayoutValidationTest.java | 221 +++++++++++++++++++++ 5 files changed, 251 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 89c5610..08e95ed 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.20 + * validate value sizes in LegacyLayout (CASSANDRA-15373) * Ensure that tracing doesn't break connections in 3.x/4.0 mixed mode by default (CASSANDRA-15385) * Make sure index summary redistribution does not start when compactions are paused (CASSANDRA-15265) * Ensure legacy rows have primary key livenessinfo when they contain illegal cells (CASSANDRA-15365) diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java index 6e93d08..c2d715d 100644 --- a/src/java/org/apache/cassandra/db/LegacyLayout.java +++ b/src/java/org/apache/cassandra/db/LegacyLayout.java @@ -198,6 +198,10 @@ public abstract class LegacyLayout } ByteBuffer collectionElement = metadata.isCompound() ? CompositeType.extractComponent(cellname, metadata.comparator.size() + 1) : null; + if (collectionElement != null && def.type instanceof CollectionType) + { + ((CollectionType)def.type).nameComparator().validateIfFixedSize(collectionElement); + } // Note that because static compact columns are translated to static defs in the new world order, we need to force a static // clustering if the definition is static (as it might not be in this case). @@ -223,6 +227,7 @@ public abstract class LegacyLayout { // The non compound case is a lot easier, in that there is no EOC nor collection to worry about, so dealing // with that first. + metadata.comparator.subtype(0).validateIfFixedSize(bound); return new LegacyBound(isStart ? Slice.Bound.inclusiveStartOf(bound) : Slice.Bound.inclusiveEndOf(bound), false, null); } @@ -231,6 +236,10 @@ public abstract class LegacyLayout boolean isStatic = metadata.isCompound() && CompositeType.isStaticName(bound); List<ByteBuffer> components = CompositeType.splitName(bound); byte eoc = CompositeType.lastEOC(bound); + for (int i=0; i<Math.min(clusteringSize, components.size()); i++) + { + metadata.comparator.subtype(i).validateIfFixedSize(components.get(i)); + } // if the bound we have decoded is static, 2.2 format requires there to be N empty clusterings assert !isStatic || @@ -438,6 +447,11 @@ public abstract class LegacyLayout ? CompositeType.splitName(value) : Collections.singletonList(value); + for (int i=0; i<Math.min(csize, components.size()); i++) + { + AbstractType<?> type = metadata.comparator.subtype(i); + type.validateIfFixedSize(components.get(i)); + } return new Clustering(components.subList(0, Math.min(csize, components.size())).toArray(new ByteBuffer[csize])); } @@ -769,6 +783,7 @@ public abstract class LegacyLayout continue; foundOne = true; + cell.name.column.type.validateIfFixedSize(cell.value); builder.addCell(new BufferCell(cell.name.column, cell.timestamp, cell.ttl, cell.localDeletionTime, cell.value, null)); } else @@ -1420,6 +1435,7 @@ public abstract class LegacyLayout if (!helper.includes(path)) return true; } + column.type.validateIfFixedSize(cell.value); Cell c = new BufferCell(column, cell.timestamp, cell.ttl, cell.localDeletionTime, cell.value, path); if (!helper.isDropped(c, column.isComplex())) builder.addCell(c); @@ -1592,6 +1608,11 @@ public abstract class LegacyLayout this.collectionElement = collectionElement; } + public static LegacyCellName create(Clustering clustering, ColumnDefinition column) + { + return new LegacyCellName(clustering, column, null); + } + public ByteBuffer encode(CFMetaData metadata) { return encodeCellName(metadata, clustering, column == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : column.name.bytes, collectionElement); diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java index 20062bd..72bfa66 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java @@ -382,6 +382,14 @@ public abstract class AbstractType<T> implements Comparator<ByteBuffer> return -1; } + public void validateIfFixedSize(ByteBuffer value) + { + if (valueLengthIfFixed() < 0) + return; + + validate(value); + } + // This assumes that no empty values are passed public void writeValue(ByteBuffer value, DataOutputPlus out) throws IOException { diff --git a/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java b/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java index 77197d3..7f3b424 100644 --- a/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java +++ b/test/unit/org/apache/cassandra/db/LegacyLayoutTest.java @@ -370,5 +370,4 @@ public class LegacyLayoutTest LegacyLayout.fromUnfilteredRowIterator(null, p.unfilteredIterator()); LegacyLayout.serializedSizeAsLegacyPartition(null, p.unfilteredIterator(), VERSION_21); } - } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java b/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java new file mode 100644 index 0000000..068d2a2 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/LegacyLayoutValidationTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.Iterator; + +import com.google.common.collect.Iterators; +import org.junit.Test; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.LegacyLayout.LegacyCell; +import org.apache.cassandra.db.LegacyLayout.LegacyCellName; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.serializers.MarshalException; + +import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes; + +public class LegacyLayoutValidationTest +{ + static final String KEYSPACE = "ks"; + + private static final CFMetaData FIXED = CFMetaData.Builder.create("ks", "cf") + .addPartitionKey("k", Int32Type.instance) + .addClusteringColumn("c1", Int32Type.instance) + .addClusteringColumn("c2", Int32Type.instance) + .addRegularColumn("v1", Int32Type.instance) + .addRegularColumn("v2", Int32Type.instance) + .build(); + + private static final CFMetaData COMPACT_FIXED = CFMetaData.Builder.create("ks", "cf", true, false, false) + .addPartitionKey("k", Int32Type.instance) + .addClusteringColumn("c", Int32Type.instance) + .addRegularColumn("v", Int32Type.instance) + .build(); + + private static final CFMetaData VARIABLE = CFMetaData.Builder.create("ks", "cf") + .addPartitionKey("k", Int32Type.instance) + .addClusteringColumn("c1", UTF8Type.instance) + .addClusteringColumn("c2", UTF8Type.instance) + .addRegularColumn("v1", UTF8Type.instance) + .addRegularColumn("v2", UTF8Type.instance) + .build(); + + private static final CFMetaData COMPACT_VARIABLE = CFMetaData.Builder.create("ks", "cf", true, false, false) + .addPartitionKey("k", Int32Type.instance) + .addClusteringColumn("c", UTF8Type.instance) + .addRegularColumn("v", UTF8Type.instance) + .build(); + + @Test + public void fixedClusteringSuccess() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)); + ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering); + LegacyLayout.decodeClustering(FIXED, serialized); + } + + @Test (expected = MarshalException.class) + public void fixedClusteringFailure() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), hexToBytes("07000000000001")); + ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering); + LegacyLayout.decodeClustering(FIXED, serialized); + } + + @Test + public void variableClusteringSuccess() + { + Clustering clustering = new Clustering(UTF8Type.instance.decompose("one"), UTF8Type.instance.decompose("two,three")); + ByteBuffer serialized = LegacyLayout.encodeClustering(VARIABLE, clustering); + LegacyLayout.decodeClustering(VARIABLE, serialized); + } + + @Test + public void fixedCompactClusteringSuccess() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(2)); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering); + LegacyLayout.decodeClustering(COMPACT_FIXED, serialized); + } + + @Test (expected = MarshalException.class) + public void fixedCompactClusteringFailure() + { + Clustering clustering = new Clustering(hexToBytes("07000000000001")); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering); + LegacyLayout.decodeClustering(COMPACT_FIXED, serialized); + } + + @Test + public void variableCompactClusteringSuccess() + { + Clustering clustering = new Clustering(UTF8Type.instance.decompose("two,three")); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_VARIABLE, clustering); + LegacyLayout.decodeClustering(COMPACT_VARIABLE, serialized); + } + + @Test + public void fixedBoundSuccess() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)); + ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering); + LegacyLayout.decodeSliceBound(FIXED, serialized, true); + } + + @Test (expected = MarshalException.class) + public void fixedBoundFailure() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), hexToBytes("07000000000001")); + ByteBuffer serialized = LegacyLayout.encodeClustering(FIXED, clustering); + LegacyLayout.decodeSliceBound(FIXED, serialized, true); + } + + @Test + public void variableBoundSuccess() + { + Clustering clustering = new Clustering(UTF8Type.instance.decompose("one"), UTF8Type.instance.decompose("two,three")); + ByteBuffer serialized = LegacyLayout.encodeClustering(VARIABLE, clustering); + LegacyLayout.decodeSliceBound(VARIABLE, serialized, true); + } + + @Test + public void fixedCompactBoundSuccess() + { + Clustering clustering = new Clustering(Int32Type.instance.decompose(1)); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering); + LegacyLayout.decodeSliceBound(COMPACT_FIXED, serialized, true); + } + + @Test (expected = MarshalException.class) + public void fixedCompactBoundFailure() + { + Clustering clustering = new Clustering(hexToBytes("07000000000001")); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_FIXED, clustering); + LegacyLayout.decodeSliceBound(COMPACT_FIXED, serialized, true); + } + + @Test + public void variableCompactBoundSuccess() + { + Clustering clustering = new Clustering(UTF8Type.instance.decompose("one")); + ByteBuffer serialized = LegacyLayout.encodeClustering(COMPACT_VARIABLE, clustering); + LegacyLayout.decodeSliceBound(COMPACT_VARIABLE, serialized, true); + } + + private static LegacyCell cell(CFMetaData cfm, Clustering clustering, String name, ByteBuffer value) throws UnknownColumnException + { + ColumnDefinition definition = cfm.getColumnDefinition(new ColumnIdentifier(name, false)); + + ByteBuffer cellName = LegacyCellName.create(clustering, definition).encode(cfm); + return LegacyCell.regular(cfm, null, cellName, value, 0); + + } + + @Test + public void fixedValueSuccess() throws Throwable + { + DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000)); + LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live(); + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)); + Iterator<LegacyCell> cells = Iterators.forArray(cell(FIXED, clustering, "v1", Int32Type.instance.decompose(3)), + cell(FIXED, clustering, "v2", Int32Type.instance.decompose(4))); + try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(FIXED, dk, deletionInfo, cells)) + { + while (iter.hasNext()) + iter.next(); + } + } + + @Test (expected = MarshalException.class) + public void fixedValueFailure() throws Throwable + { + DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000)); + LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live(); + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)); + Iterator<LegacyCell> cells = Iterators.forArray(cell(FIXED, clustering, "v1", Int32Type.instance.decompose(3)), + cell(FIXED, clustering, "v2", hexToBytes("0000"))); + try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(FIXED, dk, deletionInfo, cells)) + { + while (iter.hasNext()) + iter.next(); + } + } + + @Test + public void variableValueSuccess() throws Throwable + { + DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(1000000)); + LegacyLayout.LegacyDeletionInfo deletionInfo = LegacyLayout.LegacyDeletionInfo.live(); + Clustering clustering = new Clustering(Int32Type.instance.decompose(1), Int32Type.instance.decompose(2)); + Iterator<LegacyCell> cells = Iterators.forArray(cell(VARIABLE, clustering, "v1", UTF8Type.instance.decompose("3")), + cell(VARIABLE, clustering, "v2", hexToBytes("0000"))); + try (UnfilteredRowIterator iter = LegacyLayout.toUnfilteredRowIterator(VARIABLE, dk, deletionInfo, cells)) + { + while (iter.hasNext()) + iter.next(); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org