http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftConversion.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java index adb925e..148fd68 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java +++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java @@ -23,31 +23,35 @@ import java.util.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.CachingOptions; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.ColumnFamilyType; -import org.apache.cassandra.schema.LegacySchemaTables; -import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.schema.LegacySchemaTables; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDGen; /** * Static utility methods to convert internal structure to and from thrift ones. + * + * */ public class ThriftConversion { + private static final Logger logger = LoggerFactory.getLogger(ThriftConversion.class); + public static org.apache.cassandra.db.ConsistencyLevel fromThrift(ConsistencyLevel cl) { switch (cl) @@ -136,21 +140,14 @@ public class ThriftConversion return new TimedOutException(); } - public static List<org.apache.cassandra.db.IndexExpression> indexExpressionsFromThrift(List<IndexExpression> exprs) + public static RowFilter rowFilterFromThrift(CFMetaData metadata, List<IndexExpression> exprs) { - if (exprs == null) - return null; - - if (exprs.isEmpty()) - return Collections.emptyList(); + if (exprs == null || exprs.isEmpty()) + return RowFilter.NONE; - List<org.apache.cassandra.db.IndexExpression> converted = new ArrayList<>(exprs.size()); + RowFilter converted = RowFilter.forThrift(exprs.size()); for (IndexExpression expr : exprs) - { - converted.add(new org.apache.cassandra.db.IndexExpression(expr.column_name, - Operator.valueOf(expr.op.name()), - expr.value)); - } + converted.addThriftExpression(metadata, expr.column_name, Operator.valueOf(expr.op.name()), expr.value); return converted; } @@ -184,50 +181,75 @@ public class ThriftConversion public static CFMetaData fromThrift(CfDef cf_def) throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException { - return internalFromThrift(cf_def, Collections.<ColumnDefinition>emptyList()); + // This is a creation: the table is dense if it doesn't define any column_metadata + boolean isDense = cf_def.column_metadata == null || cf_def.column_metadata.isEmpty(); + return internalFromThrift(cf_def, true, Collections.<ColumnDefinition>emptyList(), isDense); } public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate) throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException { - return internalFromThrift(cf_def, toUpdate.allColumns()); + return internalFromThrift(cf_def, false, toUpdate.allColumns(), toUpdate.isDense()); } - // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild - private static CFMetaData internalFromThrift(CfDef cf_def, Collection<ColumnDefinition> previousCQLMetadata) - throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException + private static boolean isSuper(String thriftColumnType) + throws org.apache.cassandra.exceptions.InvalidRequestException { - ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); - if (cfType == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + cf_def.column_type); + switch (thriftColumnType.toLowerCase()) + { + case "standard": return false; + case "super": return true; + default: throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + thriftColumnType); + } + } + /** + * Convert a thrift CfDef. + * <p>, + * This is used both for creation and update of CF. + * + * @param cf_def the thrift CfDef to convert. + * @param isCreation whether that is a new table creation or not. + * @param previousCQLMetadata if it is not a table creation, the previous + * definitions of the tables (which we use to preserve the CQL metadata). + * If it is a table creation, this will be empty. + * @param isDense whether the table is dense or not. + * + * @return the converted table definition. + */ + private static CFMetaData internalFromThrift(CfDef cf_def, + boolean isCreation, + Collection<ColumnDefinition> previousCQLMetadata, + boolean isDense) + throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException + { applyImplicitDefaults(cf_def); try { + boolean isSuper = isSuper(cf_def.column_type); AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type); - AbstractType<?> subComparator = cfType == ColumnFamilyType.Standard - ? null - : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type); + AbstractType<?> subComparator = isSuper + ? cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type) + : null; - AbstractType<?> fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator); + AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : BytesType.instance; + AbstractType<?> defaultValidator = TypeParser.parse(cf_def.default_validation_class); - AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null; - - // Convert the REGULAR definitions from the input CfDef + // Convert the definitions from the input CfDef List<ColumnDefinition> defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata); - // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for + // Add the keyAlias if there is one, since that's a CQL metadata that thrift can actually change (for // historical reasons) boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType); if (hasKeyAlias) - defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null)); + defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, UTF8Type.instance.getString(cf_def.key_alias), keyValidator, null)); // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one for (ColumnDefinition def : previousCQLMetadata) { // isPartOfCellName basically means 'is not just a CQL metadata' - if (def.isPartOfCellName()) + if (def.isPartOfCellName(false, isSuper)) continue; if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias) @@ -236,18 +258,25 @@ public class ThriftConversion defs.add(def); } - CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, CFMetaData.calculateIsDense(fullRawComparator, defs)); - UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name); if (cfId == null) cfId = UUIDGen.getTimeUUID(); - CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId); + boolean isCompound = isSuper ? false : (rawComparator instanceof CompositeType); + boolean isCounter = defaultValidator instanceof CounterColumnType; - newCFMD.addAllColumnDefinitions(defs); + // If it's a thrift table creation, adds the default CQL metadata for the new table + if (isCreation) + addDefaultCQLMetadata(defs, + cf_def.keyspace, + cf_def.name, + hasKeyAlias ? null : keyValidator, + rawComparator, + subComparator, + defaultValidator); + + CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, defs); - if (keyValidator != null) - newCFMD.keyValidator(keyValidator); if (cf_def.isSetGc_grace_seconds()) newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); if (cf_def.isSetMin_compaction_threshold()) @@ -280,9 +309,7 @@ public class ThriftConversion newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers)); return newCFMD.comment(cf_def.comment) - .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) - .compressionParameters(CompressionParameters.create(cf_def.compression_options)) - .rebuild(); + .compressionParameters(CompressionParameters.create(cf_def.compression_options)); } catch (SyntaxException | MarshalException e) { @@ -290,6 +317,48 @@ public class ThriftConversion } } + private static void addDefaultCQLMetadata(Collection<ColumnDefinition> defs, + String ks, + String cf, + AbstractType<?> keyValidator, + AbstractType<?> comparator, + AbstractType<?> subComparator, + AbstractType<?> defaultValidator) + { + CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs); + if (keyValidator != null) + { + if (keyValidator instanceof CompositeType) + { + List<AbstractType<?>> subTypes = ((CompositeType)keyValidator).types; + for (int i = 0; i < subTypes.size(); i++) + defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), subTypes.get(i), i)); + } + else + { + defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), keyValidator, null)); + } + } + + if (subComparator != null) + { + // SuperColumn tables: we use a special map to hold dynamic values within a given super column + defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), comparator, 0)); + defs.add(ColumnDefinition.regularDef(ks, cf, CompactTables.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true), null)); + } + else + { + List<AbstractType<?>> subTypes = comparator instanceof CompositeType + ? ((CompositeType)comparator).types + : Collections.<AbstractType<?>>singletonList(comparator); + + for (int i = 0; i < subTypes.size(); i++) + defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), subTypes.get(i), i)); + + defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator, null)); + } + } + /** applies implicit defaults to cf definition. useful in updates */ private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) { @@ -355,30 +424,30 @@ public class ThriftConversion public static CfDef toThrift(CFMetaData cfm) { CfDef def = new CfDef(cfm.ksName, cfm.cfName); - def.setColumn_type(cfm.cfType.name()); + def.setColumn_type(cfm.isSuper() ? "Super" : "Standard"); if (cfm.isSuper()) { def.setComparator_type(cfm.comparator.subtype(0).toString()); - def.setSubcomparator_type(cfm.comparator.subtype(1).toString()); + def.setSubcomparator_type(cfm.thriftColumnNameType().toString()); } else { - def.setComparator_type(cfm.comparator.toString()); + def.setComparator_type(LegacyLayout.makeLegacyComparator(cfm).toString()); } def.setComment(Strings.nullToEmpty(cfm.getComment())); def.setRead_repair_chance(cfm.getReadRepairChance()); def.setDclocal_read_repair_chance(cfm.getDcLocalReadRepairChance()); def.setGc_grace_seconds(cfm.getGcGraceSeconds()); - def.setDefault_validation_class(cfm.getDefaultValidator().toString()); + def.setDefault_validation_class(cfm.makeLegacyDefaultValidator().toString()); def.setKey_validation_class(cfm.getKeyValidator().toString()); def.setMin_compaction_threshold(cfm.getMinCompactionThreshold()); def.setMax_compaction_threshold(cfm.getMaxCompactionThreshold()); // We only return the alias if only one is set since thrift don't know about multiple key aliases if (cfm.partitionKeyColumns().size() == 1) def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes); - def.setColumn_metadata(columnDefinitionsToThrift(cfm.allColumns())); + def.setColumn_metadata(columnDefinitionsToThrift(cfm, cfm.allColumns())); def.setCompaction_strategy(cfm.compactionStrategyClass.getName()); def.setCompaction_strategy_options(new HashMap<>(cfm.compactionStrategyOptions)); def.setCompression_options(cfm.compressionParameters.asThriftOptions()); @@ -402,8 +471,9 @@ public class ThriftConversion ColumnDef thriftColumnDef) throws SyntaxException, ConfigurationException { + boolean isSuper = thriftSubcomparator != null; // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component. - Integer componentIndex = thriftSubcomparator != null ? 1 : null; + Integer componentIndex = isSuper ? 1 : null; AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator; try { @@ -414,15 +484,18 @@ public class ThriftConversion throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator)); } + // In our generic layout, we store thrift defined columns as static, but this doesn't work for super columns so we + // use a regular definition (and "dynamic" columns are handled in a map). + ColumnDefinition.Kind kind = isSuper ? ColumnDefinition.Kind.REGULAR : ColumnDefinition.Kind.STATIC; return new ColumnDefinition(ksName, cfName, - new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator), + ColumnIdentifier.getInterned(ByteBufferUtil.clone(thriftColumnDef.name), comparator), TypeParser.parse(thriftColumnDef.validation_class), thriftColumnDef.index_type == null ? null : org.apache.cassandra.config.IndexType.valueOf(thriftColumnDef.index_type.name()), thriftColumnDef.index_options, thriftColumnDef.index_name, componentIndex, - ColumnDefinition.Kind.REGULAR); + kind); } private static List<ColumnDefinition> fromThrift(String ksName, @@ -456,11 +529,11 @@ public class ThriftConversion return cd; } - private static List<ColumnDef> columnDefinitionsToThrift(Collection<ColumnDefinition> columns) + private static List<ColumnDef> columnDefinitionsToThrift(CFMetaData metadata, Collection<ColumnDefinition> columns) { List<ColumnDef> thriftDefs = new ArrayList<>(columns.size()); for (ColumnDefinition def : columns) - if (def.kind == ColumnDefinition.Kind.REGULAR) + if (def.isPartOfCellName(metadata.isCQLTable(), metadata.isSuper())) thriftDefs.add(ThriftConversion.toThrift(def)); return thriftDefs; }
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java new file mode 100644 index 0000000..ccb6e74 --- /dev/null +++ b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.thrift; + +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.partitions.*; + +/** + * Given an iterator on a partition of a compact table, this return an iterator that merges the + * static row columns with the other results. + * + * Compact tables stores thrift column_metadata as static columns (see CompactTables for + * details). When reading for thrift however, we want to merge those static values with other + * results because: + * 1) on thrift, all "columns" are sorted together, whether or not they are declared + * column_metadata. + * 2) it's possible that a table add a value for a "dynamic" column, and later that column + * is statically defined. Merging "static" and "dynamic" columns make sure we don't miss + * a value prior to the column declaration. + * + * For example, if a thrift table declare 2 columns "c1" and "c5" and the results from a query + * is: + * Partition: static: { c1: 3, c5: 4 } + * "a" : { value : 2 } + * "c3": { value : 8 } + * "c7": { value : 1 } + * then this class transform it into: + * Partition: "a" : { value : 2 } + * "c1": { value : 3 } + * "c3": { value : 8 } + * "c5": { value : 4 } + * "c7": { value : 1 } + */ +public class ThriftResultsMerger extends WrappingUnfilteredPartitionIterator +{ + private final int nowInSec; + + private ThriftResultsMerger(UnfilteredPartitionIterator wrapped, int nowInSec) + { + super(wrapped); + this.nowInSec = nowInSec; + } + + public static UnfilteredPartitionIterator maybeWrap(UnfilteredPartitionIterator iterator, CFMetaData metadata, int nowInSec) + { + if (!metadata.isStaticCompactTable() && !metadata.isSuper()) + return iterator; + + return new ThriftResultsMerger(iterator, nowInSec); + } + + public static UnfilteredRowIterator maybeWrap(UnfilteredRowIterator iterator, int nowInSec) + { + if (!iterator.metadata().isStaticCompactTable() && !iterator.metadata().isSuper()) + return iterator; + + return iterator.metadata().isSuper() + ? new SuperColumnsPartitionMerger(iterator, nowInSec) + : new PartitionMerger(iterator, nowInSec); + } + + protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter) + { + return iter.metadata().isSuper() + ? new SuperColumnsPartitionMerger(iter, nowInSec) + : new PartitionMerger(iter, nowInSec); + } + + private static class PartitionMerger extends WrappingUnfilteredRowIterator + { + private final int nowInSec; + + // We initialize lazily to avoid having this iterator fetch the wrapped iterator before it's actually asked for it. + private boolean isInit; + + private Row staticRow; + private int i; // the index of the next column of static row to return + + private ReusableRow nextToMerge; + private Unfiltered nextFromWrapped; + + private PartitionMerger(UnfilteredRowIterator results, int nowInSec) + { + super(results); + assert results.metadata().isStaticCompactTable(); + this.nowInSec = nowInSec; + } + + private void init() + { + assert !isInit; + this.staticRow = super.staticRow(); + assert staticRow.columns().complexColumnCount() == 0; + + this.nextToMerge = createReusableRow(); + updateNextToMerge(); + isInit = true; + } + + @Override + public Row staticRow() + { + return Rows.EMPTY_STATIC_ROW; + } + + private ReusableRow createReusableRow() + { + return new ReusableRow(metadata().clusteringColumns().size(), metadata().partitionColumns().regulars, true, metadata().isCounter()); + } + + @Override + public boolean hasNext() + { + if (!isInit) + init(); + + return nextFromWrapped != null || nextToMerge != null || super.hasNext(); + } + + @Override + public Unfiltered next() + { + if (!isInit) + init(); + + if (nextFromWrapped == null && super.hasNext()) + nextFromWrapped = super.next(); + + if (nextFromWrapped == null) + { + if (nextToMerge == null) + throw new NoSuchElementException(); + + return consumeNextToMerge(); + } + + if (nextToMerge == null) + return consumeNextWrapped(); + + int cmp = metadata().comparator.compare(nextToMerge, nextFromWrapped); + if (cmp < 0) + return consumeNextToMerge(); + if (cmp > 0) + return consumeNextWrapped(); + + // Same row, but we know the row has only a single column so just pick the more recent + assert nextFromWrapped instanceof Row; + ReusableRow row = createReusableRow(); + Rows.merge((Row)consumeNextWrapped(), consumeNextToMerge(), columns().regulars, row.writer(), nowInSec); + return row; + } + + private Unfiltered consumeNextWrapped() + { + Unfiltered toReturn = nextFromWrapped; + nextFromWrapped = null; + return toReturn; + } + + private Row consumeNextToMerge() + { + Row toReturn = nextToMerge; + updateNextToMerge(); + return toReturn; + } + + private void updateNextToMerge() + { + while (i < staticRow.columns().simpleColumnCount()) + { + Cell cell = staticRow.getCell(staticRow.columns().getSimple(i++)); + if (cell != null) + { + // Given a static cell, the equivalent row uses the column name as clustering and the + // value as unique cell value. + Row.Writer writer = nextToMerge.writer(); + writer.writeClusteringValue(cell.column().name.bytes); + writer.writeCell(metadata().compactValueColumn(), cell.isCounterCell(), cell.value(), cell.livenessInfo(), cell.path()); + writer.endOfRow(); + return; + } + } + // Nothing more to merge. + nextToMerge = null; + } + } + + private static class SuperColumnsPartitionMerger extends WrappingUnfilteredRowIterator + { + private final int nowInSec; + private final ReusableRow reusableRow; + private final ColumnDefinition superColumnMapColumn; + private final AbstractType<?> columnComparator; + + private SuperColumnsPartitionMerger(UnfilteredRowIterator results, int nowInSec) + { + super(results); + assert results.metadata().isSuper(); + this.nowInSec = nowInSec; + + this.superColumnMapColumn = results.metadata().compactValueColumn(); + assert superColumnMapColumn != null && superColumnMapColumn.type instanceof MapType; + + this.reusableRow = new ReusableRow(results.metadata().clusteringColumns().size(), + Columns.of(superColumnMapColumn), + true, + results.metadata().isCounter()); + this.columnComparator = ((MapType)superColumnMapColumn.type).nameComparator(); + } + + @Override + public Unfiltered next() + { + Unfiltered next = super.next(); + if (next.kind() != Unfiltered.Kind.ROW) + return next; + + Row row = (Row)next; + Row.Writer writer = reusableRow.writer(); + row.clustering().writeTo(writer); + + PeekingIterator<Cell> staticCells = Iterators.peekingIterator(makeStaticCellIterator(row)); + if (!staticCells.hasNext()) + return row; + + Iterator<Cell> cells = row.getCells(superColumnMapColumn); + PeekingIterator<Cell> dynamicCells = Iterators.peekingIterator(cells.hasNext() ? cells : Collections.<Cell>emptyIterator()); + + while (staticCells.hasNext() && dynamicCells.hasNext()) + { + Cell staticCell = staticCells.peek(); + Cell dynamicCell = dynamicCells.peek(); + int cmp = columnComparator.compare(staticCell.column().name.bytes, dynamicCell.path().get(0)); + if (cmp < 0) + { + staticCell = staticCells.next(); + writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes)); + } + else if (cmp > 0) + { + dynamicCells.next().writeTo(writer); + } + else + { + staticCell = staticCells.next(); + Cell toMerge = Cells.create(superColumnMapColumn, + staticCell.isCounterCell(), + staticCell.value(), + staticCell.livenessInfo(), + CellPath.create(staticCell.column().name.bytes)); + Cells.reconcile(toMerge, dynamicCells.next(), nowInSec).writeTo(writer); + } + } + + while (staticCells.hasNext()) + { + Cell staticCell = staticCells.next(); + writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes)); + } + while (dynamicCells.hasNext()) + { + dynamicCells.next().writeTo(writer); + } + + writer.endOfRow(); + return reusableRow; + } + + private static Iterator<Cell> makeStaticCellIterator(final Row row) + { + return new AbstractIterator<Cell>() + { + private int i; + + protected Cell computeNext() + { + while (i < row.columns().simpleColumnCount()) + { + Cell cell = row.getCell(row.columns().getSimple(i++)); + if (cell != null) + return cell; + } + return endOfData(); + } + }; + } + } +} + http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftValidation.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java index 3768952..dd5bf98 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java +++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java @@ -24,17 +24,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.*; -import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.serializers.MarshalException; @@ -123,7 +116,7 @@ public class ThriftValidation */ public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) { if (column_path.super_column != null) { @@ -151,7 +144,7 @@ public class ThriftValidation public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) { if (column_parent.super_column != null) { @@ -168,19 +161,19 @@ public class ThriftValidation // column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException { - if (metadata.cfType == ColumnFamilyType.Standard) + if (metadata.isSuper()) { - if (column_path_or_parent.super_column != null) + if (column_path_or_parent.super_column == null && column_path_or_parent.column != null) { - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName); + throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF " + + metadata.cfName); } } - if (metadata.cfType == ColumnFamilyType.Super) + else { - if (column_path_or_parent.super_column == null && column_path_or_parent.column != null) + if (column_path_or_parent.super_column != null) { - throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF " - + metadata.cfName); + throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName); } } if (column_path_or_parent.column != null) @@ -193,13 +186,30 @@ public class ThriftValidation } } + private static AbstractType<?> getThriftColumnNameComparator(CFMetaData metadata, ByteBuffer superColumnName) + { + if (!metadata.isSuper()) + return LegacyLayout.makeLegacyComparator(metadata); + + if (superColumnName == null) + { + // comparator for super column name + return metadata.comparator.subtype(0); + } + else + { + // comparator for sub columns + return metadata.thriftColumnNameType(); + } + } + /** * Validates the column names but not the parent path or data */ private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names) throws org.apache.cassandra.exceptions.InvalidRequestException { - int maxNameLength = Cell.MAX_NAME_LENGTH; + int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH; if (superColumnName != null) { @@ -207,10 +217,10 @@ public class ThriftValidation throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength); if (superColumnName.remaining() == 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty"); - if (metadata.cfType == ColumnFamilyType.Standard) + if (!metadata.isSuper()) throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns"); } - AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, superColumnName); + AbstractType<?> comparator = getThriftColumnNameComparator(metadata, superColumnName); boolean isCQL3Table = !metadata.isThriftCompatible(); for (ByteBuffer name : column_names) { @@ -229,31 +239,26 @@ public class ThriftValidation if (isCQL3Table) { - // CQL3 table don't support having only part of their composite column names set - Composite composite = metadata.comparator.fromByteBuffer(name); - - int minComponents = metadata.comparator.clusteringPrefixSize() + 1; - if (composite.size() < minComponents) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table", - composite.size(), minComponents, metadata.cfName)); - - // Furthermore, the column name must be a declared one. - int columnIndex = metadata.comparator.clusteringPrefixSize(); - ByteBuffer CQL3ColumnName = composite.get(columnIndex); - if (!CQL3ColumnName.hasRemaining()) - continue; // Row marker, ok - - ColumnIdentifier columnId = new ColumnIdentifier(CQL3ColumnName, metadata.comparator.subtype(columnIndex)); - if (metadata.getColumnDefinition(columnId) == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid cell for CQL3 table %s. The CQL3 column component (%s) does not correspond to a defined CQL3 column", - metadata.cfName, columnId)); - - // On top of that, if we have a collection component, he (CQL3) column must be a collection - if (metadata.comparator.hasCollections() && composite.size() == metadata.comparator.size()) + try + { + LegacyLayout.LegacyCellName cname = LegacyLayout.decodeCellName(metadata, name); + assert cname.clustering.size() == metadata.comparator.size(); + + // CQL3 table don't support having only part of their composite column names set + for (int i = 0; i < cname.clustering.size(); i++) + { + if (cname.clustering.get(i) == null) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table", + i, metadata.comparator.size() + 1, metadata.cfName)); + } + + // On top of that, if we have a collection component, the (CQL3) column must be a collection + if (cname.column != null && cname.collectionElement != null && !cname.column.type.isCollection()) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", cname.column.name)); + } + catch (IllegalArgumentException | UnknownColumnException e) { - ColumnToCollectionType collectionType = metadata.comparator.collectionType(); - if (!collectionType.defined.containsKey(CQL3ColumnName)) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", UTF8Type.instance.getString(CQL3ColumnName))); + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Error validating cell name for CQL3 table %s: %s", metadata.cfName, e.getMessage())); } } } @@ -269,13 +274,13 @@ public class ThriftValidation if (range.count < 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count"); - int maxNameLength = Cell.MAX_NAME_LENGTH; + int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH; if (range.start.remaining() > maxNameLength) throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength); if (range.finish.remaining() > maxNameLength) throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength); - AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, column_parent.super_column); + AbstractType<?> comparator = getThriftColumnNameComparator(metadata, column_parent.super_column); try { comparator.validate(range.start); @@ -295,7 +300,7 @@ public class ThriftValidation } } - public static void validateColumnOrSuperColumn(CFMetaData metadata, ByteBuffer key, ColumnOrSuperColumn cosc) + public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc) throws org.apache.cassandra.exceptions.InvalidRequestException { boolean isCommutative = metadata.isCounter(); @@ -316,7 +321,7 @@ public class ThriftValidation validateTtl(cosc.column); validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name)); - validateColumnData(metadata, key, null, cosc.column); + validateColumnData(metadata, null, cosc.column); } if (cosc.super_column != null) @@ -327,7 +332,7 @@ public class ThriftValidation for (Column c : cosc.super_column.columns) { validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name)); - validateColumnData(metadata, key, cosc.super_column.name, c); + validateColumnData(metadata, cosc.super_column.name, c); } } @@ -356,8 +361,8 @@ public class ThriftValidation if (column.ttl <= 0) throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be positive"); - if (column.ttl > ExpiringCell.MAX_TTL) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, ExpiringCell.MAX_TTL)); + if (column.ttl > Attributes.MAX_TTL) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, Attributes.MAX_TTL)); } else { @@ -366,7 +371,7 @@ public class ThriftValidation } } - public static void validateMutation(CFMetaData metadata, ByteBuffer key, Mutation mut) + public static void validateMutation(CFMetaData metadata, Mutation mut) throws org.apache.cassandra.exceptions.InvalidRequestException { ColumnOrSuperColumn cosc = mut.column_or_supercolumn; @@ -383,7 +388,7 @@ public class ThriftValidation if (cosc != null) { - validateColumnOrSuperColumn(metadata, key, cosc); + validateColumnOrSuperColumn(metadata, cosc); } else { @@ -400,7 +405,7 @@ public class ThriftValidation if (del.predicate != null) validateSlicePredicate(metadata, del.super_column, del.predicate); - if (metadata.cfType == ColumnFamilyType.Standard && del.super_column != null) + if (!metadata.isSuper() && del.super_column != null) { String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del); throw new org.apache.cassandra.exceptions.InvalidRequestException(msg); @@ -409,7 +414,7 @@ public class ThriftValidation if (metadata.isCounter()) { // forcing server timestamp even if a timestamp was set for coherence with other counter operation - del.timestamp = System.currentTimeMillis(); + del.timestamp = FBUtilities.timestampMicros(); } else if (!del.isSetTimestamp()) { @@ -432,7 +437,7 @@ public class ThriftValidation /** * Validates the data part of the column (everything in the column object but the name, which is assumed to be valid) */ - public static void validateColumnData(CFMetaData metadata, ByteBuffer key, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException + public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException { validateTtl(column); if (!column.isSetValue()) @@ -440,14 +445,17 @@ public class ThriftValidation if (!column.isSetTimestamp()) throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required"); - CellName cn = scName == null - ? metadata.comparator.cellFromByteBuffer(column.name) - : metadata.comparator.makeCellName(scName, column.name); try { - AbstractType<?> validator = metadata.getValueValidator(cn); - if (validator != null) - validator.validate(column.value); + LegacyLayout.LegacyCellName cn = LegacyLayout.decodeCellName(metadata, scName, column.name); + cn.column.validateCellValue(column.value); + + // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details + Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(cn.column, column.value, null); + } + catch (UnknownColumnException e) + { + throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); } catch (MarshalException me) { @@ -458,25 +466,9 @@ public class ThriftValidation me.getMessage(), metadata.ksName, metadata.cfName, - (SuperColumns.getComparatorFor(metadata, scName != null)).getString(column.name))); + (getThriftColumnNameComparator(metadata, scName)).getString(column.name))); } - // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details - SecondaryIndex failedIndex = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(key, asDBColumn(cn, column)); - if (failedIndex != null) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s", - column.value.remaining(), - failedIndex.getIndexName(), - metadata.cfName, - metadata.ksName)); - } - - private static Cell asDBColumn(CellName name, Column column) - { - if (column.ttl <= 0) - return new BufferCell(name, column.value, column.timestamp); - else - return new BufferExpiringCell(name, column.value, column.timestamp, column.ttl); } /** @@ -535,8 +527,8 @@ public class ThriftValidation else if (range.start_key != null && range.end_token != null) { // start_token/end_token can wrap, but key/token should not - RowPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound(); - if (RowPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum()) + PartitionPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound(); + if (PartitionPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum()) throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token"); } @@ -577,7 +569,7 @@ public class ThriftValidation return false; SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager; - AbstractType<?> nameValidator = SuperColumns.getComparatorFor(metadata, null); + AbstractType<?> nameValidator = getThriftColumnNameComparator(metadata, null); boolean isIndexed = false; for (IndexExpression expression : index_clause) @@ -597,11 +589,18 @@ public class ThriftValidation if (expression.value.remaining() > 0xFFFF) throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K"); - CellName name = metadata.comparator.cellFromByteBuffer(expression.column_name); - AbstractType<?> valueValidator = metadata.getValueValidator(name); + ColumnDefinition def = metadata.getColumnDefinition(expression.column_name); + if (def == null) + { + if (!metadata.isCompactTable()) + throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Unknown column %s", nameValidator.getString(expression.column_name))); + + def = metadata.compactValueColumn(); + } + try { - valueValidator.validate(expression.value); + def.type.validate(expression.value); } catch (MarshalException me) { @@ -611,7 +610,7 @@ public class ThriftValidation me.getMessage())); } - isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(name); + isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(def); } return isIndexed; @@ -637,32 +636,32 @@ public class ThriftValidation throw new org.apache.cassandra.exceptions.InvalidRequestException("system keyspace is not user-modifiable"); } - public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn) - { - SliceRange sr = sp.slice_range; - IDiskAtomFilter filter; - - CellNameType comparator = metadata.isSuper() - ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1)) - : metadata.comparator; - if (sr == null) - { - - SortedSet<CellName> ss = new TreeSet<CellName>(comparator); - for (ByteBuffer bb : sp.column_names) - ss.add(comparator.cellFromByteBuffer(bb)); - filter = new NamesQueryFilter(ss); - } - else - { - filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start), - comparator.fromByteBuffer(sr.finish), - sr.reversed, - sr.count); - } - - if (metadata.isSuper()) - filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter); - return filter; - } + //public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn) + //{ + // SliceRange sr = sp.slice_range; + // IDiskAtomFilter filter; + + // CellNameType comparator = metadata.isSuper() + // ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1)) + // : metadata.comparator; + // if (sr == null) + // { + + // SortedSet<CellName> ss = new TreeSet<CellName>(comparator); + // for (ByteBuffer bb : sp.column_names) + // ss.add(comparator.cellFromByteBuffer(bb)); + // filter = new NamesQueryFilter(ss); + // } + // else + // { + // filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start), + // comparator.fromByteBuffer(sr.finish), + // sr.reversed, + // sr.count); + // } + + // if (metadata.isSuper()) + // filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter); + // return filter; + //} } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/tools/SSTableExport.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java deleted file mode 100644 index 9f833e7..0000000 --- a/src/java/org/apache/cassandra/tools/SSTableExport.java +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.tools; - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.*; - -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.cli.*; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.map.ObjectMapper; - -/** - * Export SSTables to JSON format. - */ -public class SSTableExport -{ - private static final ObjectMapper jsonMapper = new ObjectMapper(); - - private static final String KEY_OPTION = "k"; - private static final String EXCLUDEKEY_OPTION = "x"; - private static final String ENUMERATEKEYS_OPTION = "e"; - - private static final Options options = new Options(); - private static CommandLine cmd; - - static - { - Option optKey = new Option(KEY_OPTION, true, "Row key"); - // Number of times -k <key> can be passed on the command line. - optKey.setArgs(500); - options.addOption(optKey); - - Option excludeKey = new Option(EXCLUDEKEY_OPTION, true, "Excluded row key"); - // Number of times -x <key> can be passed on the command line. - excludeKey.setArgs(500); - options.addOption(excludeKey); - - Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only"); - options.addOption(optEnumerate); - - // disabling auto close of the stream - jsonMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); - } - - /** - * Checks if PrintStream error and throw exception - * - * @param out The PrintStream to be check - */ - private static void checkStream(PrintStream out) throws IOException - { - if (out.checkError()) - throw new IOException("Error writing output stream"); - } - - /** - * JSON Hash Key serializer - * - * @param out The output steam to write data - * @param value value to set as a key - */ - private static void writeKey(PrintStream out, String value) - { - writeJSON(out, value); - out.print(": "); - } - - private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData) - { - if (atom instanceof Cell) - { - return serializeColumn((Cell) atom, cfMetaData); - } - else - { - assert atom instanceof RangeTombstone; - RangeTombstone rt = (RangeTombstone) atom; - ArrayList<Object> serializedColumn = new ArrayList<Object>(); - serializedColumn.add(cfMetaData.comparator.getString(rt.min)); - serializedColumn.add(cfMetaData.comparator.getString(rt.max)); - serializedColumn.add(rt.data.markedForDeleteAt); - serializedColumn.add("t"); - serializedColumn.add(rt.data.localDeletionTime); - return serializedColumn; - } - } - - /** - * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Type is - * - * human_readable_name, value, timestamp, [flag, [options]] - * - * Value is normally the human readable value as rendered by the validator, but for deleted cells we - * give the local deletion time instead. - * - * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter: - * - No options for deleted cells - * - If expiring, options will include the TTL and local deletion time. - * - If counter, options will include timestamp of last delete - * - * @param cell cell presentation - * @param cfMetaData Column Family metadata (to get validator) - * @return cell as serialized list - */ - private static List<Object> serializeColumn(Cell cell, CFMetaData cfMetaData) - { - CellNameType comparator = cfMetaData.comparator; - ArrayList<Object> serializedColumn = new ArrayList<Object>(); - - serializedColumn.add(comparator.getString(cell.name())); - - if (cell instanceof DeletedCell) - { - serializedColumn.add(cell.getLocalDeletionTime()); - } - else - { - AbstractType<?> validator = cfMetaData.getValueValidator(cell.name()); - serializedColumn.add(validator.getString(cell.value())); - } - - serializedColumn.add(cell.timestamp()); - - if (cell instanceof DeletedCell) - { - serializedColumn.add("d"); - } - else if (cell instanceof ExpiringCell) - { - serializedColumn.add("e"); - serializedColumn.add(((ExpiringCell) cell).getTimeToLive()); - serializedColumn.add(cell.getLocalDeletionTime()); - } - else if (cell instanceof CounterCell) - { - serializedColumn.add("c"); - serializedColumn.add(((CounterCell) cell).timestampOfLastDelete()); - } - - return serializedColumn; - } - - /** - * Get portion of the columns and serialize in loop while not more columns left in the row - * - * @param row SSTableIdentityIterator row representation with Column Family - * @param key Decorated Key for the required row - * @param out output stream - */ - private static void serializeRow(SSTableIdentityIterator row, DecoratedKey key, PrintStream out) - { - serializeRow(row.getColumnFamily().deletionInfo(), row, row.getColumnFamily().metadata(), key, out); - } - - private static void serializeRow(DeletionInfo deletionInfo, Iterator<OnDiskAtom> atoms, CFMetaData metadata, DecoratedKey key, PrintStream out) - { - out.print("{"); - writeKey(out, "key"); - writeJSON(out, metadata.getKeyValidator().getString(key.getKey())); - out.print(",\n"); - - if (!deletionInfo.isLive()) - { - out.print(" "); - writeKey(out, "metadata"); - out.print("{"); - writeKey(out, "deletionInfo"); - writeJSON(out, deletionInfo.getTopLevelDeletion()); - out.print("}"); - out.print(",\n"); - } - - out.print(" "); - writeKey(out, "cells"); - out.print("["); - while (atoms.hasNext()) - { - writeJSON(out, serializeAtom(atoms.next(), metadata)); - - if (atoms.hasNext()) - out.print(",\n "); - } - out.print("]"); - - out.print("}"); - } - - /** - * Enumerate row keys from an SSTableReader and write the result to a PrintStream. - * - * @param desc the descriptor of the file to export the rows from - * @param outs PrintStream to write the output to - * @param metadata Metadata to print keys in a proper format - * @throws IOException on failure to read/write input/output - */ - public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata) - throws IOException - { - try (KeyIterator iter = new KeyIterator(desc)) - { - DecoratedKey lastKey = null; - while (iter.hasNext()) - { - DecoratedKey key = iter.next(); - - // validate order of the keys in the sstable - if (lastKey != null && lastKey.compareTo(key) > 0) - throw new IOException("Key out of order! " + lastKey + " > " + key); - lastKey = key; - - outs.println(metadata.getKeyValidator().getString(key.getKey())); - checkStream(outs); // flushes - } - } - } - - /** - * Export specific rows from an SSTable and write the resulting JSON to a PrintStream. - * - * @param desc the descriptor of the sstable to read from - * @param outs PrintStream to write the output to - * @param toExport the keys corresponding to the rows to export - * @param excludes keys to exclude from export - * @param metadata Metadata to print keys in a proper format - * @throws IOException on failure to read/write input/output - */ - public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes, CFMetaData metadata) throws IOException - { - SSTableReader sstable = SSTableReader.open(desc); - - try (RandomAccessReader dfile = sstable.openDataReader()) - { - IPartitioner partitioner = sstable.partitioner; - - if (excludes != null) - toExport.removeAll(Arrays.asList(excludes)); - - outs.println("["); - - int i = 0; - - // last key to compare order - DecoratedKey lastKey = null; - - for (String key : toExport) - { - DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key)); - - if (lastKey != null && lastKey.compareTo(decoratedKey) > 0) - throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey); - - lastKey = decoratedKey; - - RowIndexEntry entry = sstable.getPosition(decoratedKey, SSTableReader.Operator.EQ); - if (entry == null) - continue; - - dfile.seek(entry.position); - ByteBufferUtil.readWithShortLength(dfile); // row key - DeletionInfo deletionInfo = new DeletionInfo(DeletionTime.serializer.deserialize(dfile)); - - Iterator<OnDiskAtom> atomIterator = sstable.metadata.getOnDiskIterator(dfile, sstable.descriptor.version); - checkStream(outs); - - if (i != 0) - outs.println(","); - i++; - serializeRow(deletionInfo, atomIterator, sstable.metadata, decoratedKey, outs); - } - - outs.println("\n]"); - outs.flush(); - } - } - - // This is necessary to accommodate the test suite since you cannot open a Reader more - // than once from within the same process. - static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException - { - Set<String> excludeSet = new HashSet<String>(); - - if (excludes != null) - excludeSet = new HashSet<>(Arrays.asList(excludes)); - - SSTableIdentityIterator row; - ISSTableScanner scanner = reader.getScanner(); - try - { - outs.println("["); - - int i = 0; - - // collecting keys to export - while (scanner.hasNext()) - { - row = (SSTableIdentityIterator) scanner.next(); - - String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey()); - - if (excludeSet.contains(currentKey)) - continue; - else if (i != 0) - outs.println(","); - - serializeRow(row, row.getKey(), outs); - checkStream(outs); - - i++; - } - - outs.println("\n]"); - outs.flush(); - } - finally - { - scanner.close(); - } - } - - /** - * Export an SSTable and write the resulting JSON to a PrintStream. - * - * @param desc the descriptor of the sstable to read from - * @param outs PrintStream to write the output to - * @param excludes keys to exclude from export - * @throws IOException on failure to read/write input/output - */ - public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException - { - export(SSTableReader.open(desc), outs, excludes); - } - - /** - * Export an SSTable and write the resulting JSON to standard out. - * - * @param desc the descriptor of the sstable to read from - * @param excludes keys to exclude from export - * @throws IOException on failure to read/write SSTable/standard out - */ - public static void export(Descriptor desc, String[] excludes) throws IOException - { - export(desc, System.out, excludes); - } - - /** - * Given arguments specifying an SSTable, and optionally an output file, - * export the contents of the SSTable to JSON. - * - * @param args command lines arguments - * @throws ConfigurationException on configuration failure (wrong params given) - */ - public static void main(String[] args) throws ConfigurationException - { - System.err.println("WARNING: please note that sstable2json is now deprecated and will be removed in Cassandra 3.0. " - + "Please see https://issues.apache.org/jira/browse/CASSANDRA-9618 for details."); - - String usage = String.format("Usage: %s <sstable> [-k key [-k key [...]] -x key [-x key [...]]]%n", SSTableExport.class.getName()); - - CommandLineParser parser = new PosixParser(); - try - { - cmd = parser.parse(options, args); - } - catch (ParseException e1) - { - System.err.println(e1.getMessage()); - System.err.println(usage); - System.exit(1); - } - - - if (cmd.getArgs().length != 1) - { - System.err.println("You must supply exactly one sstable"); - System.err.println(usage); - System.exit(1); - } - - - String[] keys = cmd.getOptionValues(KEY_OPTION); - String[] excludes = cmd.getOptionValues(EXCLUDEKEY_OPTION); - String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath(); - - Schema.instance.loadFromDisk(false); - Descriptor descriptor = Descriptor.fromFilename(ssTableFileName); - - // Start by validating keyspace name - if (Schema.instance.getKSMetaData(descriptor.ksname) == null) - { - System.err.println(String.format("Filename %s references to nonexistent keyspace: %s!", - ssTableFileName, descriptor.ksname)); - System.exit(1); - } - Keyspace keyspace = Keyspace.open(descriptor.ksname); - - // Make it works for indexes too - find parent cf if necessary - String baseName = descriptor.cfname; - if (descriptor.cfname.contains(".")) - { - String[] parts = descriptor.cfname.split("\\.", 2); - baseName = parts[0]; - } - - // IllegalArgumentException will be thrown here if ks/cf pair does not exist - ColumnFamilyStore cfStore = null; - try - { - cfStore = keyspace.getColumnFamilyStore(baseName); - } - catch (IllegalArgumentException e) - { - System.err.println(String.format("The provided table is not part of this cassandra keyspace: keyspace = %s, table = %s", - descriptor.ksname, descriptor.cfname)); - System.exit(1); - } - - try - { - if (cmd.hasOption(ENUMERATEKEYS_OPTION)) - { - enumeratekeys(descriptor, System.out, cfStore.metadata); - } - else - { - if ((keys != null) && (keys.length > 0)) - export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata); - else - export(descriptor, excludes); - } - } - catch (IOException e) - { - // throwing exception outside main with broken pipe causes windows cmd to hang - e.printStackTrace(System.err); - } - - System.exit(0); - } - - private static void writeJSON(PrintStream out, Object value) - { - try - { - jsonMapper.writeValue(out, value); - } - catch (Exception e) - { - throw new RuntimeException(e.getMessage(), e); - } - } -}
