This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new cc3e742c73 Fix SSTabledump errors when dumping data from index cc3e742c73 is described below commit cc3e742c735daddc29d2fb90aac638c641463d39 Author: maxwellguo <cclive1...@gmail.com> AuthorDate: Mon Oct 24 21:53:50 2022 +0800 Fix SSTabledump errors when dumping data from index patch by Maxwell Guo; reviewed by Andrés de la Peña and Branimir Lambov for CASSANDRA-17698 --- CHANGES.txt | 1 + .../db/marshal/AbstractCompositeType.java | 1 + .../db/marshal/PartitionerDefinedOrder.java | 63 +++++-- .../apache/cassandra/db/marshal/TypeParser.java | 63 ++++++- .../org/apache/cassandra/dht/IPartitioner.java | 13 ++ .../apache/cassandra/dht/Murmur3Partitioner.java | 7 +- .../apache/cassandra/dht/RandomPartitioner.java | 7 +- .../cassandra/index/internal/CassandraIndex.java | 10 +- .../index/internal/CassandraIndexFunctions.java | 58 ++++++- .../apache/cassandra/tools/JsonTransformer.java | 12 +- .../org/apache/cassandra/tools/SSTableExport.java | 4 +- src/java/org/apache/cassandra/tools/Util.java | 12 ++ .../cql3/SecondaryIndexSSTableExportTest.java | 181 +++++++++++++++++++++ .../db/marshal/PartitionerDefinedOrderTest.java | 59 +++++++ .../cassandra/db/marshal/TypeParserTest.java | 132 ++++++++++++++- .../apache/cassandra/dht/LengthPartitioner.java | 5 + .../apache/cassandra/utils/FBUtilitiesTest.java | 2 +- .../bytecomparable/AbstractTypeByteSourceTest.java | 2 +- 18 files changed, 592 insertions(+), 40 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9fa2bafff2..8014ca6f70 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Fix SSTabledump errors when dumping data from index (CASSANDRA-17698) * Avoid unnecessary deserialization of terminal arguments when executing CQL functions (CASSANDRA-18566) * Remove dependency on pytz library for setting CQLSH timezones on Python version >= 3.9 (CASSANDRA-17433) * Extend maximum expiration date (CASSANDRA-14227) diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java index 18007fb711..f120349785 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java @@ -266,6 +266,7 @@ public abstract class AbstractCompositeType extends AbstractType<ByteBuffer> @Override public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion) { + // TODO: suport toJSONString (CASSANDRA-18177) throw new UnsupportedOperationException(); } diff --git a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java index be4b354ad0..57786edbb8 100644 --- a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java +++ b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java @@ -18,46 +18,54 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; -import java.util.Iterator; +import java.util.Objects; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Term; import org.apache.cassandra.cql3.functions.ArgumentDeserializer; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.serializers.TypeSerializer; -import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version; import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.FBUtilities; + +import javax.annotation.Nullable; /** for sorting columns representing row keys in the row ordering as determined by a partitioner. * Not intended for user-defined CFs, and will in fact error out if used with such. */ public class PartitionerDefinedOrder extends AbstractType<ByteBuffer> { private final IPartitioner partitioner; - + private final AbstractType<?> partitionKeyType; + public PartitionerDefinedOrder(IPartitioner partitioner) { super(ComparisonType.CUSTOM); this.partitioner = partitioner; + this.partitionKeyType = null; + } + + public PartitionerDefinedOrder(IPartitioner partitioner, AbstractType<?> partitionKeyType) + { + super(ComparisonType.CUSTOM); + this.partitioner = partitioner; + this.partitionKeyType = partitionKeyType; } public static AbstractType<?> getInstance(TypeParser parser) { - IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); - Iterator<String> argIterator = parser.getKeyValueParameters().keySet().iterator(); - if (argIterator.hasNext()) - { - partitioner = FBUtilities.newPartitioner(argIterator.next()); - assert !argIterator.hasNext(); - } - return partitioner.partitionOrdering(); + return parser.getPartitionerDefinedOrder(); } + public AbstractType<?> withPartitionKeyType(AbstractType<?> partitionKeyType) + { + return new PartitionerDefinedOrder(partitioner, partitionKeyType); + } + @Override public <V> ByteBuffer compose(V value, ValueAccessor<V> accessor) { @@ -89,7 +97,8 @@ public class PartitionerDefinedOrder extends AbstractType<ByteBuffer> @Override public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion) { - throw new UnsupportedOperationException(); + assert partitionKeyType != null : "PartitionerDefinedOrder's toJSONString method needs a partition key type but now is null."; + return partitionKeyType.toJSONString(buffer, protocolVersion); } public <VL, VR> int compareCustom(VL left, ValueAccessor<VL> accessorL, VR right, ValueAccessor<VR> accessorR) @@ -146,6 +155,32 @@ public class PartitionerDefinedOrder extends AbstractType<ByteBuffer> @Override public String toString() { + if (partitionKeyType != null && !DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5)) + { + return String.format("%s(%s:%s)", getClass().getName(), partitioner.getClass().getName(), partitionKeyType); + } + // if Cassandra's major version is before 5, use the old behaviour return String.format("%s(%s)", getClass().getName(), partitioner.getClass().getName()); } + + @Nullable + public AbstractType<?> getPartitionKeyType() + { + return partitionKeyType; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + if (obj instanceof PartitionerDefinedOrder) + { + PartitionerDefinedOrder other = (PartitionerDefinedOrder) obj; + return partitioner.equals(other.partitioner) && Objects.equals(partitionKeyType, other.partitionKeyType); + } + return false; + } } diff --git a/src/java/org/apache/cassandra/db/marshal/TypeParser.java b/src/java/org/apache/cassandra/db/marshal/TypeParser.java index 7416d49143..db8d3c43cd 100644 --- a/src/java/org/apache/cassandra/db/marshal/TypeParser.java +++ b/src/java/org/apache/cassandra/db/marshal/TypeParser.java @@ -21,13 +21,20 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import com.google.common.base.Verify; import com.google.common.collect.ImmutableMap; - +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.FieldIdentifier; -import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -131,6 +138,56 @@ public class TypeParser return getAbstractType(name); } + /** + * Parse PartitionOrdering from old version of PartitionOrdering' string format + */ + private static AbstractType<?> defaultParsePartitionOrdering(TypeParser typeParser) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Iterator<String> argIterator = typeParser.getKeyValueParameters().keySet().iterator(); + if (argIterator.hasNext()) + { + partitioner = FBUtilities.newPartitioner(argIterator.next()); + assert !argIterator.hasNext(); + } + return partitioner.partitionOrdering(null); + } + + /** + * Parse and return the real {@link PartitionerDefinedOrder} from the string variable {@link #str}. + * The {@link #str} format can be like {@code PartitionerDefinedOrder(<partitioner>)} or + * {@code PartitionerDefinedOrder(<partitioner>:<baseType>)}. + */ + public AbstractType<?> getPartitionerDefinedOrder() + { + int initIdx = idx; + skipBlank(); + if (isEOS()) + return defaultParsePartitionOrdering(this); + if (str.charAt(idx) != '(') + throw new IllegalStateException(); + + ++idx; // skipping '(' + skipBlank(); + + String k = readNextIdentifier(); + IPartitioner partitioner = FBUtilities.newPartitioner(k); + skipBlank(); + if (str.charAt(idx) == ':') + { + ++idx; + skipBlank(); + // must be PartitionerDefinedOrder + return partitioner.partitionOrdering(parse()); + } + else if (str.charAt(idx) == ')') + { + idx = initIdx; + return partitioner.partitionOrdering(null); + } + throw new SyntaxException("Syntax error parsing '" + str + ": for msg unexpected character '" + str.charAt(idx) + "'"); + } + public Map<String, String> getKeyValueParameters() throws SyntaxException { if (isEOS()) diff --git a/src/java/org/apache/cassandra/dht/IPartitioner.java b/src/java/org/apache/cassandra/dht/IPartitioner.java index ef8ced25b1..1858d0ce35 100644 --- a/src/java/org/apache/cassandra/dht/IPartitioner.java +++ b/src/java/org/apache/cassandra/dht/IPartitioner.java @@ -28,6 +28,8 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.service.StorageService; +import javax.annotation.Nullable; + public interface IPartitioner { static IPartitioner global() @@ -129,8 +131,19 @@ public interface IPartitioner * Abstract type that orders the same way as DecoratedKeys provided by this partitioner. * Used by secondary indices. */ + @Deprecated // use #partitionOrdering(AbstractType) instead, see CASSANDRA-17698 for details public AbstractType<?> partitionOrdering(); + /** + * Abstract type that orders the same way as DecoratedKeys provided by this partitioner. + * Used by secondary indices. + * @param partitionKeyType partition key type for PartitionerDefinedOrder + */ + default AbstractType<?> partitionOrdering(@Nullable AbstractType<?> partitionKeyType) + { + return partitionOrdering(); + } + default Optional<Splitter> splitter() { return Optional.empty(); diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java index 57993a69a6..73d7b4f3ed 100644 --- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java +++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java @@ -54,7 +54,7 @@ public class Murmur3Partitioner implements IPartitioner private static final int HEAP_SIZE = (int) ObjectSizes.measureDeep(MINIMUM); public static final Murmur3Partitioner instance = new Murmur3Partitioner(); - public static final AbstractType<?> partitionOrdering = new PartitionerDefinedOrder(instance); + public static final PartitionerDefinedOrder partitionOrdering = new PartitionerDefinedOrder(instance); private final Splitter splitter = new Splitter(this) { @@ -421,6 +421,11 @@ public class Murmur3Partitioner implements IPartitioner return partitionOrdering; } + public AbstractType<?> partitionOrdering(AbstractType<?> partitionKeyType) + { + return partitionOrdering.withPartitionKeyType(partitionKeyType); + } + public Optional<Splitter> splitter() { return Optional.of(splitter); diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java index 15743016f0..930ebb1a47 100644 --- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java +++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java @@ -79,7 +79,7 @@ public class RandomPartitioner implements IPartitioner private static final int HEAP_SIZE = (int) ObjectSizes.measureDeep(new BigIntegerToken(hashToBigInteger(ByteBuffer.allocate(1)))); public static final RandomPartitioner instance = new RandomPartitioner(); - public static final AbstractType<?> partitionOrdering = new PartitionerDefinedOrder(instance); + public static final PartitionerDefinedOrder partitionOrdering = new PartitionerDefinedOrder(instance); private final Splitter splitter = new Splitter(this) { @@ -347,6 +347,11 @@ public class RandomPartitioner implements IPartitioner return partitionOrdering; } + public AbstractType<?> partitionOrdering(AbstractType<?> partitionKeyType) + { + return partitionOrdering.withPartitionKeyType(partitionKeyType); + } + public Optional<Splitter> splitter() { return Optional.of(splitter); diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java index 1c8942bc47..09ccf1aa3a 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.ColumnMetadata; @@ -143,7 +144,7 @@ public abstract class CassandraIndex implements Index Clustering<?> clustering, CellPath path, ByteBuffer cellValue); - + public ColumnMetadata getIndexedColumn() { return indexedColumn; @@ -735,12 +736,15 @@ public abstract class CassandraIndex implements Index ColumnMetadata indexedColumn = target.left; AbstractType<?> indexedValueType = utils.getIndexedValueType(indexedColumn); + // if Cassandra's major version is before 5, use the old behaviour + boolean isCompatible = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5); + AbstractType<?> indexedTablePartitionKeyType = baseCfsMetadata.partitioner.partitionOrdering(baseCfsMetadata.partitionKeyType); TableMetadata.Builder builder = TableMetadata.builder(baseCfsMetadata.keyspace, baseCfsMetadata.indexTableName(indexMetadata), baseCfsMetadata.id) .kind(TableMetadata.Kind.INDEX) .partitioner(new LocalPartitioner(indexedValueType)) - .addPartitionKeyColumn(indexedColumn.name, indexedColumn.type) - .addClusteringColumn("partition_key", baseCfsMetadata.partitioner.partitionOrdering()); + .addPartitionKeyColumn(indexedColumn.name, isCompatible ? indexedColumn.type : utils.getIndexedPartitionKeyType(indexedColumn)) + .addClusteringColumn("partition_key", isCompatible ? baseCfsMetadata.partitioner.partitionOrdering() : indexedTablePartitionKeyType); // Adding clustering columns, which depends on the index type. builder = utils.addIndexClusteringColumns(builder, baseCfsMetadata, indexedColumn); diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java b/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java index 1bba1a9a0b..e3380d0af3 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndexFunctions.java @@ -20,15 +20,23 @@ package org.apache.cassandra.index.internal; import java.util.List; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CollectionType; import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.index.internal.composites.*; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.SetType; +import org.apache.cassandra.index.internal.composites.ClusteringColumnIndex; +import org.apache.cassandra.index.internal.composites.CollectionEntryIndex; +import org.apache.cassandra.index.internal.composites.CollectionKeyIndex; +import org.apache.cassandra.index.internal.composites.CollectionValueIndex; +import org.apache.cassandra.index.internal.composites.PartitionKeyIndex; +import org.apache.cassandra.index.internal.composites.RegularColumnIndex; import org.apache.cassandra.index.internal.keys.KeysIndex; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.TableMetadata; public interface CassandraIndexFunctions { @@ -51,6 +59,11 @@ public interface CassandraIndexFunctions return indexedColumn.type; } + default AbstractType<?> getIndexedPartitionKeyType(ColumnMetadata indexedColumn) + { + return indexedColumn.type; + } + /** * Add the clustering columns for a specific type of index table to the a TableMetadata.Builder (which is being * used to construct the index table's TableMetadata. In the default implementation, the clustering columns of the @@ -149,6 +162,22 @@ public interface CassandraIndexFunctions { return ((CollectionType) indexedColumn.type).nameComparator(); } + + @Override + public AbstractType<?> getIndexedPartitionKeyType(ColumnMetadata indexedColumn) + { + assert indexedColumn.type.isCollection(); + switch (((CollectionType<?>)indexedColumn.type).kind) + { + case LIST: + return ((ListType<?>)indexedColumn.type).getElementsType(); + case SET: + return ((SetType<?>)indexedColumn.type).getElementsType(); + case MAP: + return ((MapType<?, ?>)indexedColumn.type).getKeysType(); + } + throw new RuntimeException("Error collection type " + indexedColumn.type); + } }; static final CassandraIndexFunctions PARTITION_KEY_INDEX_FUNCTIONS = new CassandraIndexFunctions() @@ -183,6 +212,22 @@ public interface CassandraIndexFunctions builder.addClusteringColumn("cell_path", ((CollectionType)columnDef.type).nameComparator()); return builder; } + + @Override + public AbstractType<?> getIndexedPartitionKeyType(ColumnMetadata indexedColumn) + { + assert indexedColumn.type.isCollection(); + switch (((CollectionType<?>) indexedColumn.type).kind) + { + case LIST: + return ((ListType<?>) indexedColumn.type).getElementsType(); + case SET: + return ((SetType<?>) indexedColumn.type).getElementsType(); + case MAP: + return ((MapType<?, ?>) indexedColumn.type).getValuesType(); + } + throw new RuntimeException("Error collection type " + indexedColumn.type); + } }; static final CassandraIndexFunctions COLLECTION_ENTRY_INDEX_FUNCTIONS = new CassandraIndexFunctions() @@ -197,5 +242,12 @@ public interface CassandraIndexFunctions CollectionType colType = (CollectionType)indexedColumn.type; return CompositeType.getInstance(colType.nameComparator(), colType.valueComparator()); } + + @Override + public AbstractType<?> getIndexedPartitionKeyType(ColumnMetadata indexedColumn) + { + assert indexedColumn.type.isCollection(); + return indexedColumn.type; + } }; } diff --git a/src/java/org/apache/cassandra/tools/JsonTransformer.java b/src/java/org/apache/cassandra/tools/JsonTransformer.java index ed0830dfa3..8debfd3b75 100644 --- a/src/java/org/apache/cassandra/tools/JsonTransformer.java +++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java @@ -38,7 +38,11 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.Indenter; import com.fasterxml.jackson.core.util.MinimalPrettyPrinter; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.LivenessInfo; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CollectionType; import org.apache.cassandra.db.marshal.CompositeType; @@ -203,7 +207,8 @@ public final class JsonTransformer try { json.writeStartObject(); - + json.writeObjectField("table kind", metadata.kind.name()); + json.writeFieldName("partition"); json.writeStartObject(); json.writeFieldName("key"); @@ -369,7 +374,8 @@ public final class JsonTransformer } else { - json.writeRawValue(column.cellValueType().toJSONString(clustering.get(i), clustering.accessor(), ProtocolVersion.CURRENT)); + AbstractType<?> type = column.cellValueType(); + json.writeRawValue(type.toJSONString(clustering.get(i), clustering.accessor(), ProtocolVersion.CURRENT)); } } json.writeEndArray(); diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java index da9298a4ca..d807e0cde9 100644 --- a/src/java/org/apache/cassandra/tools/SSTableExport.java +++ b/src/java/org/apache/cassandra/tools/SSTableExport.java @@ -50,6 +50,8 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST; + /** * Export SSTables to JSON format. */ @@ -72,7 +74,7 @@ public class SSTableExport static { - DatabaseDescriptor.toolInitialization(); + DatabaseDescriptor.toolInitialization(!TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean()); Option optKey = new Option(KEY_OPTION, true, "List of included partition keys"); // Number of times -k <key> can be passed on the command line. diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java index 8a254e25aa..d8ef121f89 100644 --- a/src/java/org/apache/cassandra/tools/Util.java +++ b/src/java/org/apache/cassandra/tools/Util.java @@ -40,9 +40,12 @@ import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.Indexes; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.FBUtilities; @@ -332,6 +335,15 @@ public final class Util { builder.addClusteringColumn("clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i)); } + if (SecondaryIndexManager.isIndexColumnFamily(desc.cfname)) + { + String index = SecondaryIndexManager.getIndexName(desc.cfname); + // Just set the Kind of index to CUSTOM, which is an irrelevant parameter that doesn't make any effect on the result + IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(index, IndexMetadata.Kind.CUSTOM, null); + Indexes indexes = Indexes.of(indexMetadata); + builder.indexes(indexes); + builder.kind(TableMetadata.Kind.INDEX); + } return builder.build(); } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/cql3/SecondaryIndexSSTableExportTest.java b/test/unit/org/apache/cassandra/cql3/SecondaryIndexSSTableExportTest.java new file mode 100644 index 0000000000..9d016cb343 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/SecondaryIndexSSTableExportTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.exc.MismatchedInputException; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.tools.SSTableExport; +import org.apache.cassandra.tools.ToolRunner; +import org.apache.cassandra.utils.Pair; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.utils.JsonUtils; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class SecondaryIndexSSTableExportTest extends CQLTester +{ + private static final TypeReference<List<Map<String, Object>>> jacksonListOfMapsType = new TypeReference<List<Map<String, Object>>>() {}; + private static boolean initValue; + + @BeforeClass + public static void beforeClass() + { + initValue = TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean(); + TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.setBoolean(true); + } + + @AfterClass + public static void afterClass() + { + TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.setBoolean(initValue); + } + + @Test + public void testRegularColumnIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int PRIMARY KEY, v int)"; + String createIndex = "CREATE INDEX ON %s (v)"; + String insert = "INSERT INTO %s (k, v) VALUES (0, 0)"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testPartitionKeyIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int, v int, c text, primary key((k, v)))"; + String createIndex = "CREATE INDEX ON %s (k)"; + String insert = "INSERT INTO %s (k, v) VALUES (0, 0)"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testKeysWithStaticIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, primary key(k, v))"; + String createIndex = "CREATE INDEX ON %s (v)"; + String insert = "INSERT INTO %s (k, v, s) VALUES (0, 0, 's')"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testClusteringIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, c bigint, primary key((k, v), c))"; + String createIndex = "CREATE INDEX ON %s (c)"; + String insert = "INSERT INTO %s (k, v, s, c) VALUES (0, 0, 's', 10)"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testCollectionMapKeyIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, c bigint, m map<bigint, text>, l list<text>, st set<int>, primary key((k, v), c))"; + String createIndex = "CREATE INDEX ON %s (KEYS(m))"; + String insert = "INSERT INTO %s (k, v, s, c, m, l, st) VALUES (0, 0, 's', 10, {100:'v'}, ['l1', 'l2'], {1, 2, 3})"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testCollectionMapValueIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, c bigint, m map<bigint, text>, l list<text>, st set<int>, primary key((k, v), c))"; + String createIndex = "CREATE INDEX ON %s (VALUES(m))"; + String insert = "INSERT INTO %s (k, v, s, c, m, l, st) VALUES (0, 0, 's', 10, {100:'v'}, ['l1', 'l2'], {1, 2, 3})"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testCollectionListIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, c bigint, m map<bigint, text>, l list<text>, st set<int>, primary key((k, v), c))"; + String createIndex = "CREATE INDEX ON %s (l)"; + String insert = "INSERT INTO %s (k, v, s, c, m, l, st) VALUES (0, 0, 's', 10, {100:'v'}, ['l1', 'l2'], {1, 2, 3})"; + indexSstableValidation(createTable, createIndex, insert); + } + + @Test + public void testCollectionSetIndex() throws Throwable + { + String createTable = "CREATE TABLE %s (k int , v int, s text static, c bigint, m map<bigint, text>, l list<text>, st set<int>, primary key((k, v), c))"; + String createIndex = "CREATE INDEX ON %s (st)"; + String insert = "INSERT INTO %s (k, v, s, c, m, l, st) VALUES (0, 0, 's', 10, {100:'v'}, ['l1', 'l2'], {1, 2, 3})"; + indexSstableValidation(createTable, createIndex, insert); + } + + private void indexSstableValidation(String createTableCql, String createIndexCql, String insertCql) throws Throwable + { + Pair<String, String> tableIndex = generateSstable(createTableCql, createIndexCql, insertCql); + ColumnFamilyStore cfs = getColumnFamilyStore(KEYSPACE, tableIndex.left); + assertTrue(cfs.indexManager.hasIndexes()); + assertNotNull(cfs.indexManager.getIndexByName(tableIndex.right)); + for (ColumnFamilyStore columnFamilyStore : cfs.indexManager.getAllIndexColumnFamilyStores()) + { + assertTrue(columnFamilyStore.isIndex()); + assertFalse(columnFamilyStore.getLiveSSTables().isEmpty()); + for (SSTableReader sst : columnFamilyStore.getLiveSSTables()) + { + String file = sst.getFilename(); + try + { + ToolRunner.ToolResult tool = ToolRunner.invokeClass(SSTableExport.class, file); + List<Map<String, Object>> parsed = JsonUtils.JSON_OBJECT_MAPPER.readValue(tool.getStdout(), jacksonListOfMapsType); + assertNotNull(tool.getStdout(), parsed.get(0).get("partition")); + assertNotNull(tool.getStdout(), parsed.get(0).get("rows")); + } + catch (AssertionError e) + { + // TODO: CASSANDRA-18254 should provide a workaround for pre-5.0 sstables + assertTrue(DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5)); + Assertions.assertThat(e.getMessage()) + .contains("PartitionerDefinedOrder's toJSONString method needs a partition key type but now is null."); + } + catch (MismatchedInputException e) + { + // TODO: CASSANDRA-18254 should provide a workaround for pre-5.0 sstables + assertTrue(DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5)); + Assertions.assertThat(e.getMessage()) + .contains("No content to map due to end-of-input"); + } + } + } + } + + private Pair<String, String> generateSstable(String createTableCql, String createIndexCql, String insertCql) throws Throwable + { + String table = createTable(createTableCql); + String index = createIndex(createIndexCql); + execute(insertCql); + flush(); + return Pair.create(table, index); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/marshal/PartitionerDefinedOrderTest.java b/test/unit/org/apache/cassandra/db/marshal/PartitionerDefinedOrderTest.java new file mode 100644 index 0000000000..efd6ba5b6c --- /dev/null +++ b/test/unit/org/apache/cassandra/db/marshal/PartitionerDefinedOrderTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.marshal; + +import org.apache.cassandra.transport.ProtocolVersion; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class PartitionerDefinedOrderTest +{ + private static final String key = "key"; + private static final AbstractType<?> type = UTF8Type.instance; + + @Test + public void testToJsonStringWithBaseType() + { + TypeParserTest.assertForEachPartitioner(partitioner -> { + if (partitioner.partitionOrdering(null) instanceof PartitionerDefinedOrder) + { + PartitionerDefinedOrder partitionerDefinedOrder = (PartitionerDefinedOrder) partitioner.partitionOrdering(null); + String jsonString = partitionerDefinedOrder.withPartitionKeyType(type).toJSONString(UTF8Type.instance.decompose(key), ProtocolVersion.V4); + assertTrue(jsonString.contains(key)); + } + }); + } + + @Test + public void testToJsonStringWithOutBaseType() + { + TypeParserTest.assertForEachPartitioner(partitioner -> { + if (partitioner.partitionOrdering(null) instanceof PartitionerDefinedOrder) + { + PartitionerDefinedOrder partitionerDefinedOrder = (PartitionerDefinedOrder) partitioner.partitionOrdering(null); + assertNull(partitionerDefinedOrder.getPartitionKeyType()); + Assertions.assertThatThrownBy(() -> partitionerDefinedOrder.toJSONString(UTF8Type.instance.decompose(key), ProtocolVersion.V4)) + .hasMessageContaining("PartitionerDefinedOrder's toJSONString method needs a partition key type but now is null."); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java index ecfe910d1f..0247681c1d 100644 --- a/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/TypeParserTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,13 @@ */ package org.apache.cassandra.db.marshal; +import com.google.common.collect.Lists; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.apache.cassandra.config.DatabaseDescriptor; @@ -29,6 +32,8 @@ import org.apache.cassandra.dht.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; +import java.util.function.Consumer; + public class TypeParserTest { @BeforeClass @@ -95,14 +100,123 @@ public class TypeParserTest @Test public void testParsePartitionerOrder() throws ConfigurationException, SyntaxException { - for (IPartitioner partitioner: new IPartitioner[] { Murmur3Partitioner.instance, - ByteOrderedPartitioner.instance, - RandomPartitioner.instance, - OrderPreservingPartitioner.instance }) + assertForEachPartitioner(partitioner -> { + AbstractType<?> type = partitioner.partitionOrdering(null); + assertEquals(type, TypeParser.parse(type.toString())); + }); + assertEquals(DatabaseDescriptor.getPartitioner().partitionOrdering(null), TypeParser.parse("PartitionerDefinedOrder")); + } + + @Test + public void testParsePartitionerOrderWithBaseType() + { + // default partitioner + assertEquals(DatabaseDescriptor.getPartitioner().partitionOrdering(null), TypeParser.parse("PartitionerDefinedOrder")); + + // PartitionerDefinedOrder's base type is not composite type + differentBaseTypeValidation(Int32Type.instance); + // PartitionerDefinedOrder's base type is composite type + differentBaseTypeValidation(CompositeType.getInstance(Int32Type.instance, UTF8Type.instance)); + // PartitionerDefinedOrder's base type is tuple type + differentBaseTypeValidation(new TupleType(Lists.newArrayList(Int32Type.instance, UTF8Type.instance))); + // PartitionerDefinedOrder's base type is ReversedType + differentBaseTypeValidation(ReversedType.getInstance(Int32Type.instance)); + // PartitionerDefinedOrder's base type is CollectionType + differentBaseTypeValidation(MapType.getInstance(Int32Type.instance, UTF8Type.instance, false)); + } + + @Test + public void testParsePartitionerOrderMistMatch() + { + assertForEachPartitioner(partitioner -> { + AbstractType<?> type = partitioner.partitionOrdering(null); + if (type instanceof PartitionerDefinedOrder && !DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5)) + { + PartitionerDefinedOrder tmp = (PartitionerDefinedOrder) type; + type = tmp.withPartitionKeyType(Int32Type.instance); + boolean result = partitioner.partitionOrdering(null).equals(TypeParser.parse(type.toString())); + assertFalse(result); + } + else + { + // ByteOrderedPartitioner.instance and OrderPreservingPartitioner.instance's partitionOrdering will not be PartitionerDefinedOrder + boolean result = partitioner.partitionOrdering(null).equals(TypeParser.parse(type.toString())); + assertTrue(result); + } + }); + assertEquals(DatabaseDescriptor.getPartitioner().partitionOrdering(null), TypeParser.parse("PartitionerDefinedOrder")); + } + + @Test + public void testParsePartitionerOrderWithErrorFormat() + { + assertForEachPartitioner(partitioner -> { + AbstractType<?> type = partitioner.partitionOrdering(null); + if (type instanceof PartitionerDefinedOrder) + { + // only Murmur3Partitioner and RandomPartitioner's partitionOrdering() are instanceof PartitionerDefinedOrder + String msgPartitioner = partitioner instanceof Murmur3Partitioner ? "Murmur3Partitioner" : "RandomPartitioner"; + // error format PartitionerDefinedOrder(org.apache.cassandra.dht.Murmur3Partitioner, + String tmpStr1 = type.toString().replace(')', ','); + try + { + TypeParser.parse(tmpStr1); + fail(); + } + catch (Throwable t) + { + assertTrue(t.getCause().getMessage().contains("Syntax error parsing 'org.apache.cassandra.db.marshal.PartitionerDefinedOrder(org.apache.cassandra.dht." + msgPartitioner + ",: for msg unexpected character ','")); + } + + // error format PartitionerDefinedOrder(org.apache.cassandra.dht.Murmur3Partitioner> + String tmpStr2 = type.toString().replace(')', '>'); + try + { + TypeParser.parse(tmpStr2); + fail(); + } + catch (Throwable t) + { + assertTrue(t.getCause().getMessage().contains("Syntax error parsing 'org.apache.cassandra.db.marshal.PartitionerDefinedOrder(org.apache.cassandra.dht." + msgPartitioner + ">: for msg unexpected character '>'")); + } + + // error format PartitionerDefinedOrder(org.apache.cassandra.dht.Murmur3Partitioner> + String tmpStr3 = type.toString().replace(')', ':'); + try + { + TypeParser.parse(tmpStr3); + fail(); + } + catch (Throwable t) + { + assertTrue(t.getCause().getMessage().contains("Unable to find abstract-type class 'org.apache.cassandra.db.marshal.'")); + } + } + }); + assertEquals(DatabaseDescriptor.getPartitioner().partitionOrdering(null), TypeParser.parse("PartitionerDefinedOrder")); + } + + private void differentBaseTypeValidation(AbstractType<?> baseType) + { + assertForEachPartitioner(partitioner -> { + AbstractType<?> type = partitioner.partitionOrdering(null); + if (type instanceof PartitionerDefinedOrder && !DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5)) + { + PartitionerDefinedOrder tmp = (PartitionerDefinedOrder) type; + type = tmp.withPartitionKeyType(baseType); + } + assertEquals(type, TypeParser.parse(type.toString())); + }); + } + + public static void assertForEachPartitioner(Consumer<IPartitioner> consumer) + { + for (IPartitioner partitioner : new IPartitioner[] { Murmur3Partitioner.instance, + ByteOrderedPartitioner.instance, + RandomPartitioner.instance, + OrderPreservingPartitioner.instance }) { - AbstractType<?> type = partitioner.partitionOrdering(); - assertSame(type, TypeParser.parse(type.toString())); + consumer.accept(partitioner); } - assertSame(DatabaseDescriptor.getPartitioner().partitionOrdering(), TypeParser.parse("PartitionerDefinedOrder")); } } diff --git a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java index 9859487e94..24671e3856 100644 --- a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java +++ b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java @@ -179,4 +179,9 @@ public class LengthPartitioner implements IPartitioner { return new PartitionerDefinedOrder(this); } + + public AbstractType<?> partitionOrdering(AbstractType<?> partitionKeyType) + { + return new PartitionerDefinedOrder(this, partitionKeyType); + } } diff --git a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java index e0ee67cd58..1c77fffe75 100644 --- a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java +++ b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java @@ -158,7 +158,7 @@ public class FBUtilitiesTest IPartitioner partitioner = FBUtilities.newPartitioner(name, Optional.of(type)); Assert.assertTrue(String.format("%s != LocalPartitioner", partitioner.toString()), LocalPartitioner.class.isInstance(partitioner)); - Assert.assertEquals(partitioner.partitionOrdering(), type); + Assert.assertEquals(partitioner.partitionOrdering(null), type); } } diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java index a9b187dc6d..6fb70a3409 100644 --- a/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java +++ b/test/unit/org/apache/cassandra/utils/bytecomparable/AbstractTypeByteSourceTest.java @@ -687,7 +687,7 @@ public class AbstractTypeByteSourceTest ); for (IPartitioner partitioner : partitioners) { - AbstractType<?> partitionOrdering = partitioner.partitionOrdering(); + AbstractType<?> partitionOrdering = partitioner.partitionOrdering(null); Assert.assertTrue(partitionOrdering instanceof PartitionerDefinedOrder); for (ByteBuffer input : byteBuffers) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org