Repository: cassandra Updated Branches: refs/heads/trunk a67980e8e -> a94b173e2
Isolate Thrift conversion code from schema definition classes patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for CASSANDRA-8261 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a94b173e Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a94b173e Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a94b173e Branch: refs/heads/trunk Commit: a94b173e2c0b2ac230ec55e5f76de288e36d5f74 Parents: a67980e Author: Aleksey Yeschenko <[email protected]> Authored: Fri Nov 7 14:34:33 2014 +0300 Committer: Aleksey Yeschenko <[email protected]> Committed: Fri Nov 7 14:34:33 2014 +0300 ---------------------------------------------------------------------- .../org/apache/cassandra/config/CFMetaData.java | 260 ++----------- .../cassandra/config/ColumnDefinition.java | 63 ---- .../org/apache/cassandra/config/KSMetaData.java | 37 +- .../cassandra/config/TriggerDefinition.java | 38 +- .../cassandra/cql3/statements/CFPropDefs.java | 2 +- .../cql3/statements/PropertyDefinitions.java | 4 +- .../hadoop/AbstractBulkRecordWriter.java | 8 +- .../hadoop/pig/AbstractCassandraStorage.java | 2 +- .../cassandra/thrift/CassandraServer.java | 16 +- .../cassandra/thrift/ThriftConversion.java | 363 ++++++++++++++++++- .../org/apache/cassandra/tools/BulkLoader.java | 2 +- .../apache/cassandra/config/CFMetaDataTest.java | 11 +- .../cassandra/config/ColumnDefinitionTest.java | 3 +- .../config/DatabaseDescriptorTest.java | 5 +- .../cassandra/thrift/ThriftValidationTest.java | 6 +- 15 files changed, 413 insertions(+), 407 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/config/CFMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index 12808f1..a8d528e 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -50,10 +50,6 @@ import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.io.compress.LZ4Compressor; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.thrift.CfDef; -import org.apache.cassandra.thrift.CqlResult; -import org.apache.cassandra.thrift.CqlRow; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -444,7 +440,7 @@ public final class CFMetaData public CFMetaData compactionStrategyClass(Class<? extends AbstractCompactionStrategy> prop) {compactionStrategyClass = prop; return this;} public CFMetaData compactionStrategyOptions(Map<String, String> prop) {compactionStrategyOptions = prop; return this;} public CFMetaData compressionParameters(CompressionParameters prop) {compressionParameters = prop; return this;} - public CFMetaData bloomFilterFpChance(Double prop) {bloomFilterFpChance = prop; return this;} + public CFMetaData bloomFilterFpChance(double prop) {bloomFilterFpChance = prop; return this;} public CFMetaData caching(CachingOptions prop) {caching = prop; return this;} public CFMetaData minIndexInterval(int prop) {minIndexInterval = prop; return this;} public CFMetaData maxIndexInterval(int prop) {maxIndexInterval = prop; return this;} @@ -454,6 +450,7 @@ public final class CFMetaData public CFMetaData droppedColumns(Map<ColumnIdentifier, Long> cols) {droppedColumns = cols; return this;} public CFMetaData triggers(Map<String, TriggerDefinition> prop) {triggers = prop; return this;} public CFMetaData isDense(Boolean prop) {isDense = prop; return this;} + /** * Create new ColumnFamily metadata with generated random ID. * When loading from existing schema, use CFMetaData @@ -467,7 +464,7 @@ public final class CFMetaData this(keyspace, name, type, comp, UUIDGen.getTimeUUID()); } - private CFMetaData(String keyspace, String name, ColumnFamilyType type, CellNameType comp, UUID id) + public CFMetaData(String keyspace, String name, ColumnFamilyType type, CellNameType comp, UUID id) { cfId = id; ksName = keyspace; @@ -493,7 +490,7 @@ public final class CFMetaData return denseCFMetaData(keyspace, name, comp, null); } - private static AbstractType<?> makeRawAbstractType(AbstractType<?> comparator, AbstractType<?> subComparator) + public static AbstractType<?> makeRawAbstractType(AbstractType<?> comparator, AbstractType<?> subComparator) { return subComparator == null ? comparator : CompositeType.getInstance(Arrays.asList(comparator, subComparator)); } @@ -621,7 +618,7 @@ public final class CFMetaData .compactionStrategyClass(oldCFMD.compactionStrategyClass) .compactionStrategyOptions(new HashMap<>(oldCFMD.compactionStrategyOptions)) .compressionParameters(oldCFMD.compressionParameters.copy()) - .bloomFilterFpChance(oldCFMD.bloomFilterFpChance) + .bloomFilterFpChance(oldCFMD.getBloomFilterFpChance()) .caching(oldCFMD.caching) .defaultTimeToLive(oldCFMD.defaultTimeToLive) .minIndexInterval(oldCFMD.minIndexInterval) @@ -680,7 +677,7 @@ public final class CFMetaData return readRepairChance; } - public double getDcLocalReadRepair() + public double getDcLocalReadRepairChance() { return dcLocalReadRepairChance; } @@ -691,7 +688,7 @@ public final class CFMetaData if (getReadRepairChance() > chance) return ReadRepairDecision.GLOBAL; - if (getDcLocalReadRepair() > chance) + if (getDcLocalReadRepairChance() > chance) return ReadRepairDecision.DC_LOCAL; return ReadRepairDecision.NONE; @@ -886,7 +883,7 @@ public final class CFMetaData && Objects.equal(compactionStrategyClass, other.compactionStrategyClass) && Objects.equal(compactionStrategyOptions, other.compactionStrategyOptions) && Objects.equal(compressionParameters, other.compressionParameters) - && Objects.equal(bloomFilterFpChance, other.bloomFilterFpChance) + && Objects.equal(getBloomFilterFpChance(), other.getBloomFilterFpChance()) && Objects.equal(memtableFlushPeriod, other.memtableFlushPeriod) && Objects.equal(caching, other.caching) && Objects.equal(defaultTimeToLive, other.defaultTimeToLive) @@ -919,7 +916,7 @@ public final class CFMetaData .append(compactionStrategyClass) .append(compactionStrategyOptions) .append(compressionParameters) - .append(bloomFilterFpChance) + .append(getBloomFilterFpChance()) .append(memtableFlushPeriod) .append(caching) .append(defaultTimeToLive) @@ -938,179 +935,6 @@ public final class CFMetaData return def == null ? defaultValidator : def.type; } - /** applies implicit defaults to cf definition. useful in updates */ - private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) - { - if (!cf_def.isSetComment()) - cf_def.setComment(""); - if (!cf_def.isSetMin_compaction_threshold()) - cf_def.setMin_compaction_threshold(CFMetaData.DEFAULT_MIN_COMPACTION_THRESHOLD); - if (!cf_def.isSetMax_compaction_threshold()) - cf_def.setMax_compaction_threshold(CFMetaData.DEFAULT_MAX_COMPACTION_THRESHOLD); - if (cf_def.compaction_strategy == null) - cf_def.compaction_strategy = DEFAULT_COMPACTION_STRATEGY_CLASS.getSimpleName(); - if (cf_def.compaction_strategy_options == null) - cf_def.compaction_strategy_options = Collections.emptyMap(); - if (!cf_def.isSetCompression_options()) - { - cf_def.setCompression_options(new HashMap<String, String>() - {{ - if (DEFAULT_COMPRESSOR != null) - put(CompressionParameters.SSTABLE_COMPRESSION, DEFAULT_COMPRESSOR); - }}); - } - if (!cf_def.isSetDefault_time_to_live()) - cf_def.setDefault_time_to_live(CFMetaData.DEFAULT_DEFAULT_TIME_TO_LIVE); - if (!cf_def.isSetDclocal_read_repair_chance()) - cf_def.setDclocal_read_repair_chance(CFMetaData.DEFAULT_DCLOCAL_READ_REPAIR_CHANCE); - - // if index_interval was set, use that for the min_index_interval default - if (!cf_def.isSetMin_index_interval()) - { - if (cf_def.isSetIndex_interval()) - cf_def.setMin_index_interval(cf_def.getIndex_interval()); - else - cf_def.setMin_index_interval(CFMetaData.DEFAULT_MIN_INDEX_INTERVAL); - } - if (!cf_def.isSetMax_index_interval()) - { - // ensure the max is at least as large as the min - cf_def.setMax_index_interval(Math.max(cf_def.min_index_interval, CFMetaData.DEFAULT_MAX_INDEX_INTERVAL)); - } - } - - public static CFMetaData fromThrift(CfDef cf_def) throws InvalidRequestException, ConfigurationException - { - return internalFromThrift(cf_def, Collections.<ColumnDefinition>emptyList()); - } - - public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate) throws InvalidRequestException, ConfigurationException - { - return internalFromThrift(cf_def, toUpdate.allColumns()); - } - - // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild - private static CFMetaData internalFromThrift(CfDef cf_def, Collection<ColumnDefinition> previousCQLMetadata) throws InvalidRequestException, ConfigurationException - { - ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); - if (cfType == null) - throw new InvalidRequestException("Invalid column type " + cf_def.column_type); - - applyImplicitDefaults(cf_def); - - try - { - AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type); - AbstractType<?> subComparator = cfType == ColumnFamilyType.Standard - ? null - : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type); - - AbstractType<?> fullRawComparator = makeRawAbstractType(rawComparator, subComparator); - - AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null; - - // Convert the REGULAR definitions from the input CfDef - List<ColumnDefinition> defs = ColumnDefinition.fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata); - - // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for - // historical reasons) - boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType); - if (hasKeyAlias) - defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null)); - - // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one - for (ColumnDefinition def : previousCQLMetadata) - { - // isPartOfCellName basically means 'is not just a CQL metadata' - if (def.isPartOfCellName()) - continue; - - if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias) - continue; - - defs.add(def); - } - - CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, calculateIsDense(fullRawComparator, defs)); - - UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name); - if (cfId == null) - cfId = UUIDGen.getTimeUUID(); - - CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId); - - newCFMD.addAllColumnDefinitions(defs); - - if (keyValidator != null) - newCFMD.keyValidator(keyValidator); - if (cf_def.isSetGc_grace_seconds()) - newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); - if (cf_def.isSetMin_compaction_threshold()) - newCFMD.minCompactionThreshold(cf_def.min_compaction_threshold); - if (cf_def.isSetMax_compaction_threshold()) - newCFMD.maxCompactionThreshold(cf_def.max_compaction_threshold); - if (cf_def.isSetCompaction_strategy()) - newCFMD.compactionStrategyClass(createCompactionStrategy(cf_def.compaction_strategy)); - if (cf_def.isSetCompaction_strategy_options()) - newCFMD.compactionStrategyOptions(new HashMap<>(cf_def.compaction_strategy_options)); - if (cf_def.isSetBloom_filter_fp_chance()) - newCFMD.bloomFilterFpChance(cf_def.bloom_filter_fp_chance); - if (cf_def.isSetMemtable_flush_period_in_ms()) - newCFMD.memtableFlushPeriod(cf_def.memtable_flush_period_in_ms); - if (cf_def.isSetCaching() || cf_def.isSetCells_per_row_to_cache()) - newCFMD.caching(CachingOptions.fromThrift(cf_def.caching, cf_def.cells_per_row_to_cache)); - if (cf_def.isSetRead_repair_chance()) - newCFMD.readRepairChance(cf_def.read_repair_chance); - if (cf_def.isSetDefault_time_to_live()) - newCFMD.defaultTimeToLive(cf_def.default_time_to_live); - if (cf_def.isSetDclocal_read_repair_chance()) - newCFMD.dcLocalReadRepairChance(cf_def.dclocal_read_repair_chance); - if (cf_def.isSetMin_index_interval()) - newCFMD.minIndexInterval(cf_def.min_index_interval); - if (cf_def.isSetMax_index_interval()) - newCFMD.maxIndexInterval(cf_def.max_index_interval); - if (cf_def.isSetSpeculative_retry()) - newCFMD.speculativeRetry(SpeculativeRetry.fromString(cf_def.speculative_retry)); - if (cf_def.isSetTriggers()) - newCFMD.triggers(TriggerDefinition.fromThrift(cf_def.triggers)); - - return newCFMD.comment(cf_def.comment) - .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) - .compressionParameters(CompressionParameters.create(cf_def.compression_options)) - .rebuild(); - } - catch (SyntaxException | MarshalException e) - { - throw new ConfigurationException(e.getMessage()); - } - } - - /** - * Create CFMetaData from thrift {@link CqlRow} that contains columns from schema_columnfamilies. - * - * @param columnsRes CqlRow containing columns from schema_columnfamilies. - * @return CFMetaData derived from CqlRow - */ - public static CFMetaData fromThriftCqlRow(CqlRow cf, CqlResult columnsRes) - { - UntypedResultSet.Row cfRow = new UntypedResultSet.Row(convertThriftCqlRow(cf)); - - List<Map<String, ByteBuffer>> cols = new ArrayList<>(columnsRes.rows.size()); - for (CqlRow row : columnsRes.rows) - cols.add(convertThriftCqlRow(row)); - UntypedResultSet colsRow = UntypedResultSet.create(cols); - - return fromSchemaNoTriggers(cfRow, colsRow); - } - - private static Map<String, ByteBuffer> convertThriftCqlRow(CqlRow row) - { - Map<String, ByteBuffer> m = new HashMap<>(); - for (org.apache.cassandra.thrift.Column column : row.getColumns()) - m.put(UTF8Type.instance.getString(column.bufferForName()), column.value); - return m; - } - public void reload() { Row cfDefRow = SystemKeyspace.readSchemaRow(SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, ksName, cfName); @@ -1158,7 +982,7 @@ public final class CFMetaData minCompactionThreshold = cfm.minCompactionThreshold; maxCompactionThreshold = cfm.maxCompactionThreshold; - bloomFilterFpChance = cfm.bloomFilterFpChance; + bloomFilterFpChance = cfm.getBloomFilterFpChance(); caching = cfm.caching; minIndexInterval = cfm.minIndexInterval; maxIndexInterval = cfm.maxIndexInterval; @@ -1276,51 +1100,6 @@ public final class CFMetaData } } - // converts CFM to thrift CfDef - public org.apache.cassandra.thrift.CfDef toThrift() - { - org.apache.cassandra.thrift.CfDef def = new org.apache.cassandra.thrift.CfDef(ksName, cfName); - def.setColumn_type(cfType.name()); - - if (isSuper()) - { - def.setComparator_type(comparator.subtype(0).toString()); - def.setSubcomparator_type(comparator.subtype(1).toString()); - } - else - { - def.setComparator_type(comparator.toString()); - } - - def.setComment(Strings.nullToEmpty(comment)); - def.setRead_repair_chance(readRepairChance); - def.setDclocal_read_repair_chance(dcLocalReadRepairChance); - def.setGc_grace_seconds(gcGraceSeconds); - def.setDefault_validation_class(defaultValidator == null ? null : defaultValidator.toString()); - def.setKey_validation_class(keyValidator.toString()); - def.setMin_compaction_threshold(minCompactionThreshold); - def.setMax_compaction_threshold(maxCompactionThreshold); - // We only return the alias if only one is set since thrift don't know about multiple key aliases - if (partitionKeyColumns.size() == 1) - def.setKey_alias(partitionKeyColumns.get(0).name.bytes); - def.setColumn_metadata(ColumnDefinition.toThrift(columnMetadata)); - def.setCompaction_strategy(compactionStrategyClass.getName()); - def.setCompaction_strategy_options(new HashMap<>(compactionStrategyOptions)); - def.setCompression_options(compressionParameters.asThriftOptions()); - if (bloomFilterFpChance != null) - def.setBloom_filter_fp_chance(bloomFilterFpChance); - def.setMin_index_interval(minIndexInterval); - def.setMax_index_interval(maxIndexInterval); - def.setMemtable_flush_period_in_ms(memtableFlushPeriod); - def.setCaching(caching.toThriftCaching()); - def.setCells_per_row_to_cache(caching.toThriftCellsPerRow()); - def.setDefault_time_to_live(defaultTimeToLive); - def.setSpeculative_retry(speculativeRetry.toString()); - def.setTriggers(TriggerDefinition.toThrift(triggers)); - - return def; - } - /** * Returns the ColumnDefinition for {@code name}. */ @@ -1689,7 +1468,7 @@ public final class CFMetaData adder.add("key_validator", keyValidator.toString()); adder.add("min_compaction_threshold", minCompactionThreshold); adder.add("max_compaction_threshold", maxCompactionThreshold); - adder.add("bloom_filter_fp_chance", bloomFilterFpChance); + adder.add("bloom_filter_fp_chance", getBloomFilterFpChance()); adder.add("memtable_flush_period_in_ms", memtableFlushPeriod); adder.add("caching", caching.toString()); @@ -1713,8 +1492,8 @@ public final class CFMetaData adder.add("value_alias", compactValueColumn == null ? null : compactValueColumn.name.toString()); } - // Package protected for use by tests - static CFMetaData fromSchemaNoTriggers(UntypedResultSet.Row result, UntypedResultSet serializedColumnDefinitions) + @VisibleForTesting + public static CFMetaData fromSchemaNoTriggers(UntypedResultSet.Row result, UntypedResultSet serializedColumnDefinitions) { try { @@ -1756,8 +1535,6 @@ public final class CFMetaData cfm.maxCompactionThreshold(result.getInt("max_compaction_threshold")); if (result.has("comment")) cfm.comment(result.getString("comment")); - if (result.has("bloom_filter_fp_chance")) - cfm.bloomFilterFpChance(result.getDouble("bloom_filter_fp_chance")); if (result.has("memtable_flush_period_in_ms")) cfm.memtableFlushPeriod(result.getInt("memtable_flush_period_in_ms")); cfm.caching(CachingOptions.fromString(result.getString("caching"))); @@ -1777,6 +1554,11 @@ public final class CFMetaData if (result.has("max_index_interval")) cfm.maxIndexInterval(result.getInt("max_index_interval")); + if (result.has("bloom_filter_fp_chance")) + cfm.bloomFilterFpChance(result.getDouble("bloom_filter_fp_chance")); + else + cfm.bloomFilterFpChance(cfm.getBloomFilterFpChance()); + /* * The info previously hold by key_aliases, column_aliases and value_alias is now stored in columnMetadata (because 1) this * make more sense and 2) this allow to store indexing information). @@ -1829,7 +1611,7 @@ public final class CFMetaData /** * Deserialize CF metadata from low-level representation * - * @return Thrift-based metadata deserialized from schema + * @return Metadata deserialized from schema */ public static CFMetaData fromSchema(UntypedResultSet.Row result) { @@ -2128,7 +1910,7 @@ public final class CFMetaData * information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this * method does its best to infer whether the table is dense or not based on other elements. */ - private static boolean calculateIsDense(AbstractType<?> comparator, Collection<ColumnDefinition> defs) + public static boolean calculateIsDense(AbstractType<?> comparator, Collection<ColumnDefinition> defs) { /* * As said above, this method is only here because we need to deal with thrift upgrades. @@ -2247,7 +2029,7 @@ public final class CFMetaData .append("compactionStrategyClass", compactionStrategyClass) .append("compactionStrategyOptions", compactionStrategyOptions) .append("compressionParameters", compressionParameters.asThriftOptions()) - .append("bloomFilterFpChance", bloomFilterFpChance) + .append("bloomFilterFpChance", getBloomFilterFpChance()) .append("memtableFlushPeriod", memtableFlushPeriod) .append("caching", caching) .append("defaultTimeToLive", defaultTimeToLive) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/config/ColumnDefinition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/ColumnDefinition.java b/src/java/org/apache/cassandra/config/ColumnDefinition.java index e52bc7a..3f3ecd5 100644 --- a/src/java/org/apache/cassandra/config/ColumnDefinition.java +++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java @@ -22,16 +22,12 @@ import java.util.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; -import com.google.common.collect.Maps; import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.*; import org.apache.cassandra.db.composites.Composite; import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.exceptions.*; -import org.apache.cassandra.thrift.ColumnDef; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.FBUtilities.json; @@ -239,15 +235,6 @@ public class ColumnDefinition extends ColumnSpecification return kind == Kind.PARTITION_KEY || kind == Kind.CLUSTERING_COLUMN; } - public static List<ColumnDef> toThrift(Map<ByteBuffer, ColumnDefinition> columns) - { - List<ColumnDef> thriftDefs = new ArrayList<>(columns.size()); - for (ColumnDefinition def : columns.values()) - if (def.kind == ColumnDefinition.Kind.REGULAR) - thriftDefs.add(def.toThrift()); - return thriftDefs; - } - /** * Whether the name of this definition is serialized in the cell nane, i.e. whether * it's not just a non-stored CQL metadata. @@ -257,56 +244,6 @@ public class ColumnDefinition extends ColumnSpecification return kind == Kind.REGULAR || kind == Kind.STATIC; } - public ColumnDef toThrift() - { - ColumnDef cd = new ColumnDef(); - - cd.setName(ByteBufferUtil.clone(name.bytes)); - cd.setValidation_class(type.toString()); - cd.setIndex_type(indexType == null ? null : org.apache.cassandra.thrift.IndexType.valueOf(indexType.name())); - cd.setIndex_name(indexName == null ? null : indexName); - cd.setIndex_options(indexOptions == null ? null : Maps.newHashMap(indexOptions)); - - return cd; - } - - public static ColumnDefinition fromThrift(String ksName, String cfName, AbstractType<?> thriftComparator, AbstractType<?> thriftSubcomparator, ColumnDef thriftColumnDef) throws SyntaxException, ConfigurationException - { - // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component. - Integer componentIndex = thriftSubcomparator != null ? 1 : null; - AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator; - try - { - comparator.validate(thriftColumnDef.name); - } - catch (MarshalException e) - { - throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator)); - } - - return new ColumnDefinition(ksName, - cfName, - new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator), - TypeParser.parse(thriftColumnDef.validation_class), - thriftColumnDef.index_type == null ? null : IndexType.valueOf(thriftColumnDef.index_type.name()), - thriftColumnDef.index_options, - thriftColumnDef.index_name, - componentIndex, - Kind.REGULAR); - } - - public static List<ColumnDefinition> fromThrift(String ksName, String cfName, AbstractType<?> thriftComparator, AbstractType<?> thriftSubcomparator, List<ColumnDef> thriftDefs) throws SyntaxException, ConfigurationException - { - if (thriftDefs == null) - return Collections.emptyList(); - - List<ColumnDefinition> defs = new ArrayList<>(thriftDefs.size()); - for (ColumnDef thriftColumnDef : thriftDefs) - defs.add(fromThrift(ksName, cfName, thriftComparator, thriftSubcomparator, thriftColumnDef)); - - return defs; - } - /** * Drop specified column from the schema using given mutation. * http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/config/KSMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/KSMetaData.java b/src/java/org/apache/cassandra/config/KSMetaData.java index 64ac3ff..b5ea3ac 100644 --- a/src/java/org/apache/cassandra/config/KSMetaData.java +++ b/src/java/org/apache/cassandra/config/KSMetaData.java @@ -28,8 +28,6 @@ import org.apache.cassandra.db.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.*; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.thrift.CfDef; -import org.apache.cassandra.thrift.KsDef; import org.apache.cassandra.tracing.Tracing; import static org.apache.cassandra.utils.FBUtilities.*; @@ -44,7 +42,11 @@ public final class KSMetaData public final UTMetaData userTypes; - KSMetaData(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, boolean durableWrites, Iterable<CFMetaData> cfDefs) + public KSMetaData(String name, + Class<? extends AbstractReplicationStrategy> strategyClass, + Map<String, String> strategyOptions, + boolean durableWrites, + Iterable<CFMetaData> cfDefs) { this(name, strategyClass, strategyOptions, durableWrites, cfDefs, new UTMetaData()); } @@ -173,35 +175,6 @@ public final class KSMetaData return Collections.singletonMap("replication_factor", rf.toString()); } - public static KSMetaData fromThrift(KsDef ksd, CFMetaData... cfDefs) throws ConfigurationException - { - Class<? extends AbstractReplicationStrategy> cls = AbstractReplicationStrategy.getClass(ksd.strategy_class); - if (cls.equals(LocalStrategy.class)) - throw new ConfigurationException("Unable to use given strategy class: LocalStrategy is reserved for internal use."); - - return new KSMetaData(ksd.name, - cls, - ksd.strategy_options == null ? Collections.<String, String>emptyMap() : ksd.strategy_options, - ksd.durable_writes, - Arrays.asList(cfDefs)); - } - - public KsDef toThrift() - { - List<CfDef> cfDefs = new ArrayList<>(cfMetaData.size()); - for (CFMetaData cfm : cfMetaData().values()) - { - // Don't expose CF that cannot be correctly handle by thrift; see CASSANDRA-4377 for further details - if (cfm.isThriftCompatible()) - cfDefs.add(cfm.toThrift()); - } - KsDef ksdef = new KsDef(name, strategyClass.getName(), cfDefs); - ksdef.setStrategy_options(strategyOptions); - ksdef.setDurable_writes(durableWrites); - - return ksdef; - } - public Mutation toSchemaUpdate(KSMetaData newState, long modificationTimestamp) { return newState.toSchema(modificationTimestamp); http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/config/TriggerDefinition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/TriggerDefinition.java b/src/java/org/apache/cassandra/config/TriggerDefinition.java index aaaf631..df37cbc 100644 --- a/src/java/org/apache/cassandra/config/TriggerDefinition.java +++ b/src/java/org/apache/cassandra/config/TriggerDefinition.java @@ -27,13 +27,12 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.*; import org.apache.cassandra.db.composites.Composite; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.thrift.TriggerDef; public class TriggerDefinition { - private static final String TRIGGER_NAME = "trigger_name"; - private static final String TRIGGER_OPTIONS = "trigger_options"; - private static final String CLASS = "class"; + public static final String TRIGGER_NAME = "trigger_name"; + public static final String TRIGGER_OPTIONS = "trigger_options"; + public static final String CLASS = "class"; public final String name; @@ -41,7 +40,7 @@ public class TriggerDefinition // Proper trigger parametrization will be added later. public final String classOption; - TriggerDefinition(String name, String classOption) + public TriggerDefinition(String name, String classOption) { this.name = name; this.classOption = classOption; @@ -105,35 +104,6 @@ public class TriggerDefinition cf.addAtom(new RangeTombstone(prefix, prefix.end(), timestamp, ldt)); } - public static TriggerDefinition fromThrift(TriggerDef thriftDef) - { - return new TriggerDefinition(thriftDef.getName(), thriftDef.getOptions().get(CLASS)); - } - - public TriggerDef toThrift() - { - TriggerDef td = new TriggerDef(); - td.setName(name); - td.setOptions(Collections.singletonMap(CLASS, classOption)); - return td; - } - - public static Map<String, TriggerDefinition> fromThrift(List<TriggerDef> thriftDefs) - { - Map<String, TriggerDefinition> triggerDefinitions = new HashMap<>(); - for (TriggerDef thriftDef : thriftDefs) - triggerDefinitions.put(thriftDef.getName(), fromThrift(thriftDef)); - return triggerDefinitions; - } - - public static List<TriggerDef> toThrift(Map<String, TriggerDefinition> triggers) - { - List<TriggerDef> thriftDefs = new ArrayList<>(triggers.size()); - for (TriggerDefinition def : triggers.values()) - thriftDefs.add(def.toThrift()); - return thriftDefs; - } - @Override public boolean equals(Object o) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/cql3/statements/CFPropDefs.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CFPropDefs.java b/src/java/org/apache/cassandra/cql3/statements/CFPropDefs.java index aee86a8..948bc0b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CFPropDefs.java +++ b/src/java/org/apache/cassandra/cql3/statements/CFPropDefs.java @@ -171,7 +171,7 @@ public class CFPropDefs extends PropertyDefinitions cfm.comment(getString(KW_COMMENT, "")); cfm.readRepairChance(getDouble(KW_READREPAIRCHANCE, cfm.getReadRepairChance())); - cfm.dcLocalReadRepairChance(getDouble(KW_DCLOCALREADREPAIRCHANCE, cfm.getDcLocalReadRepair())); + cfm.dcLocalReadRepairChance(getDouble(KW_DCLOCALREADREPAIRCHANCE, cfm.getDcLocalReadRepairChance())); cfm.gcGraceSeconds(getInt(KW_GCGRACESECONDS, cfm.getGcGraceSeconds())); int minCompactionThreshold = toInt(KW_MINCOMPACTIONTHRESHOLD, getCompactionOptions().get(KW_MINCOMPACTIONTHRESHOLD), cfm.getMinCompactionThreshold()); int maxCompactionThreshold = toInt(KW_MAXCOMPACTIONTHRESHOLD, getCompactionOptions().get(KW_MAXCOMPACTIONTHRESHOLD), cfm.getMaxCompactionThreshold()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java index eb4f074..23cf8e1 100644 --- a/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java +++ b/src/java/org/apache/cassandra/cql3/statements/PropertyDefinitions.java @@ -94,8 +94,8 @@ public class PropertyDefinitions return (value == null) ? defaultValue : value.toLowerCase().matches("(1|true|yes)"); } - // Return a property value, typed as a Double - public Double getDouble(String key, Double defaultValue) throws SyntaxException + // Return a property value, typed as a double + public double getDouble(String key, double defaultValue) throws SyntaxException { String value = getSimple(key); if (value == null) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/hadoop/AbstractBulkRecordWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hadoop/AbstractBulkRecordWriter.java b/src/java/org/apache/cassandra/hadoop/AbstractBulkRecordWriter.java index 9ec37f4..d1a70d4 100644 --- a/src/java/org/apache/cassandra/hadoop/AbstractBulkRecordWriter.java +++ b/src/java/org/apache/cassandra/hadoop/AbstractBulkRecordWriter.java @@ -40,11 +40,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.SSTableLoader; import org.apache.cassandra.streaming.StreamState; -import org.apache.cassandra.thrift.AuthenticationRequest; -import org.apache.cassandra.thrift.Cassandra; -import org.apache.cassandra.thrift.CfDef; -import org.apache.cassandra.thrift.KsDef; -import org.apache.cassandra.thrift.TokenRange; +import org.apache.cassandra.thrift.*; import org.apache.cassandra.utils.OutputHandler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.RecordWriter; @@ -220,7 +216,7 @@ implements org.apache.hadoop.mapred.RecordWriter<K, V> { Map<String, CFMetaData> cfs = new HashMap<>(ksDef.cf_defs.size()); for (CfDef cfDef : ksDef.cf_defs) - cfs.put(cfDef.name, CFMetaData.fromThrift(cfDef)); + cfs.put(cfDef.name, ThriftConversion.fromThrift(cfDef)); knownCfs.put(ksDef.name, cfs); } break; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java index 19c049a..baef186 100644 --- a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java +++ b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java @@ -777,7 +777,7 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store for (CfDef cfDef : ksDef.cf_defs) { if (cfDef.name.equalsIgnoreCase(cf)) - return CFMetaData.fromThrift(cfDef); + return ThriftConversion.fromThrift(cfDef); } return null; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/thrift/CassandraServer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java index e9301f0..b7a1f7f 100644 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@ -1124,7 +1124,7 @@ public class CassandraServer implements Cassandra.Iface if (ksm == null) throw new NotFoundException(); - return ksm.toThrift(); + return ThriftConversion.toThrift(ksm); } public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) @@ -1186,7 +1186,7 @@ public class CassandraServer implements Cassandra.Iface now, filter, bounds, - ThriftConversion.fromThrift(range.row_filter), + ThriftConversion.indexExpressionsFromThrift(range.row_filter), range.count), consistencyLevel); } @@ -1354,7 +1354,7 @@ public class CassandraServer implements Cassandra.Iface now, filter, bounds, - ThriftConversion.fromThrift(index_clause.expressions), + ThriftConversion.indexExpressionsFromThrift(index_clause.expressions), index_clause.count); List<Row> rows = StorageProxy.getRangeSlice(command, consistencyLevel); @@ -1531,7 +1531,7 @@ public class CassandraServer implements Cassandra.Iface String keyspace = cState.getKeyspace(); cState.hasKeyspaceAccess(keyspace, Permission.CREATE); cf_def.unsetId(); // explicitly ignore any id set by client (Hector likes to set zero) - CFMetaData cfm = CFMetaData.fromThrift(cf_def); + CFMetaData cfm = ThriftConversion.fromThrift(cf_def); CFMetaData.validateCompactionOptions(cfm.compactionStrategyClass, cfm.compactionStrategyOptions); cfm.addDefaultIndexNames(); @@ -1591,7 +1591,7 @@ public class CassandraServer implements Cassandra.Iface for (CfDef cf_def : ks_def.cf_defs) { cf_def.unsetId(); // explicitly ignore any id set by client (same as system_add_column_family) - CFMetaData cfm = CFMetaData.fromThrift(cf_def); + CFMetaData cfm = ThriftConversion.fromThrift(cf_def); cfm.addDefaultIndexNames(); if (!cfm.getTriggers().isEmpty()) @@ -1599,7 +1599,7 @@ public class CassandraServer implements Cassandra.Iface cfDefs.add(cfm); } - MigrationManager.announceNewKeyspace(KSMetaData.fromThrift(ks_def, cfDefs.toArray(new CFMetaData[cfDefs.size()]))); + MigrationManager.announceNewKeyspace(ThriftConversion.fromThrift(ks_def, cfDefs.toArray(new CFMetaData[cfDefs.size()]))); return Schema.instance.getVersion().toString(); } catch (RequestValidationException e) @@ -1643,7 +1643,7 @@ public class CassandraServer implements Cassandra.Iface if (ks_def.getCf_defs() != null && ks_def.getCf_defs().size() > 0) throw new InvalidRequestException("Keyspace update must not contain any table definitions."); - MigrationManager.announceKeyspaceUpdate(KSMetaData.fromThrift(ks_def)); + MigrationManager.announceKeyspaceUpdate(ThriftConversion.fromThrift(ks_def)); return Schema.instance.getVersion().toString(); } catch (RequestValidationException e) @@ -1671,7 +1671,7 @@ public class CassandraServer implements Cassandra.Iface if (!oldCfm.isThriftCompatible()) throw new InvalidRequestException("Cannot modify CQL3 table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify CQL3 tables instead."); - CFMetaData cfm = CFMetaData.fromThriftForUpdate(cf_def, oldCfm); + CFMetaData cfm = ThriftConversion.fromThriftForUpdate(cf_def, oldCfm); CFMetaData.validateCompactionOptions(cfm.compactionStrategyClass, cfm.compactionStrategyOptions); cfm.addDefaultIndexNames(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/thrift/ThriftConversion.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java index 2aca45a..b04a091 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java +++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java @@ -17,16 +17,30 @@ */ package org.apache.cassandra.thrift; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.nio.ByteBuffer; +import java.util.*; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; + +import org.apache.cassandra.cache.CachingOptions; +import org.apache.cassandra.config.*; +import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.ColumnFamilyType; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.exceptions.RequestTimeoutException; -import org.apache.cassandra.exceptions.RequestValidationException; -import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.io.compress.CompressionParameters; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.UUIDGen; /** * Static utility methods to convert internal structure to and from thrift ones. @@ -113,7 +127,7 @@ public class ThriftConversion return toe; } - public static List<org.apache.cassandra.db.IndexExpression> fromThrift(List<IndexExpression> exprs) + public static List<org.apache.cassandra.db.IndexExpression> indexExpressionsFromThrift(List<IndexExpression> exprs) { if (exprs == null) return null; @@ -130,4 +144,337 @@ public class ThriftConversion } return converted; } + + public static KSMetaData fromThrift(KsDef ksd, CFMetaData... cfDefs) throws ConfigurationException + { + Class<? extends AbstractReplicationStrategy> cls = AbstractReplicationStrategy.getClass(ksd.strategy_class); + if (cls.equals(LocalStrategy.class)) + throw new ConfigurationException("Unable to use given strategy class: LocalStrategy is reserved for internal use."); + + return new KSMetaData(ksd.name, + cls, + ksd.strategy_options == null ? Collections.<String, String>emptyMap() : ksd.strategy_options, + ksd.durable_writes, + Arrays.asList(cfDefs)); + } + + public static KsDef toThrift(KSMetaData ksm) + { + List<CfDef> cfDefs = new ArrayList<>(ksm.cfMetaData().size()); + for (CFMetaData cfm : ksm.cfMetaData().values()) + if (cfm.isThriftCompatible()) // Don't expose CF that cannot be correctly handle by thrift; see CASSANDRA-4377 for further details + cfDefs.add(toThrift(cfm)); + + KsDef ksdef = new KsDef(ksm.name, ksm.strategyClass.getName(), cfDefs); + ksdef.setStrategy_options(ksm.strategyOptions); + ksdef.setDurable_writes(ksm.durableWrites); + + return ksdef; + } + + public static CFMetaData fromThrift(CfDef cf_def) + throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException + { + return internalFromThrift(cf_def, Collections.<ColumnDefinition>emptyList()); + } + + public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate) + throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException + { + return internalFromThrift(cf_def, toUpdate.allColumns()); + } + + // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild + private static CFMetaData internalFromThrift(CfDef cf_def, Collection<ColumnDefinition> previousCQLMetadata) + throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException + { + ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); + if (cfType == null) + throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + cf_def.column_type); + + applyImplicitDefaults(cf_def); + + try + { + AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type); + AbstractType<?> subComparator = cfType == ColumnFamilyType.Standard + ? null + : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type); + + AbstractType<?> fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator); + + AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null; + + // Convert the REGULAR definitions from the input CfDef + List<ColumnDefinition> defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata); + + // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for + // historical reasons) + boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType); + if (hasKeyAlias) + defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null)); + + // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one + for (ColumnDefinition def : previousCQLMetadata) + { + // isPartOfCellName basically means 'is not just a CQL metadata' + if (def.isPartOfCellName()) + continue; + + if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias) + continue; + + defs.add(def); + } + + CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, CFMetaData.calculateIsDense(fullRawComparator, defs)); + + UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name); + if (cfId == null) + cfId = UUIDGen.getTimeUUID(); + + CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId); + + newCFMD.addAllColumnDefinitions(defs); + + if (keyValidator != null) + newCFMD.keyValidator(keyValidator); + if (cf_def.isSetGc_grace_seconds()) + newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); + if (cf_def.isSetMin_compaction_threshold()) + newCFMD.minCompactionThreshold(cf_def.min_compaction_threshold); + if (cf_def.isSetMax_compaction_threshold()) + newCFMD.maxCompactionThreshold(cf_def.max_compaction_threshold); + if (cf_def.isSetCompaction_strategy()) + newCFMD.compactionStrategyClass(CFMetaData.createCompactionStrategy(cf_def.compaction_strategy)); + if (cf_def.isSetCompaction_strategy_options()) + newCFMD.compactionStrategyOptions(new HashMap<>(cf_def.compaction_strategy_options)); + if (cf_def.isSetBloom_filter_fp_chance()) + newCFMD.bloomFilterFpChance(cf_def.bloom_filter_fp_chance); + if (cf_def.isSetMemtable_flush_period_in_ms()) + newCFMD.memtableFlushPeriod(cf_def.memtable_flush_period_in_ms); + if (cf_def.isSetCaching() || cf_def.isSetCells_per_row_to_cache()) + newCFMD.caching(CachingOptions.fromThrift(cf_def.caching, cf_def.cells_per_row_to_cache)); + if (cf_def.isSetRead_repair_chance()) + newCFMD.readRepairChance(cf_def.read_repair_chance); + if (cf_def.isSetDefault_time_to_live()) + newCFMD.defaultTimeToLive(cf_def.default_time_to_live); + if (cf_def.isSetDclocal_read_repair_chance()) + newCFMD.dcLocalReadRepairChance(cf_def.dclocal_read_repair_chance); + if (cf_def.isSetMin_index_interval()) + newCFMD.minIndexInterval(cf_def.min_index_interval); + if (cf_def.isSetMax_index_interval()) + newCFMD.maxIndexInterval(cf_def.max_index_interval); + if (cf_def.isSetSpeculative_retry()) + newCFMD.speculativeRetry(CFMetaData.SpeculativeRetry.fromString(cf_def.speculative_retry)); + if (cf_def.isSetTriggers()) + newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers)); + + return newCFMD.comment(cf_def.comment) + .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) + .compressionParameters(CompressionParameters.create(cf_def.compression_options)) + .rebuild(); + } + catch (SyntaxException | MarshalException e) + { + throw new ConfigurationException(e.getMessage()); + } + } + + /** applies implicit defaults to cf definition. useful in updates */ + private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) + { + if (!cf_def.isSetComment()) + cf_def.setComment(""); + if (!cf_def.isSetMin_compaction_threshold()) + cf_def.setMin_compaction_threshold(CFMetaData.DEFAULT_MIN_COMPACTION_THRESHOLD); + if (!cf_def.isSetMax_compaction_threshold()) + cf_def.setMax_compaction_threshold(CFMetaData.DEFAULT_MAX_COMPACTION_THRESHOLD); + if (cf_def.compaction_strategy == null) + cf_def.compaction_strategy = CFMetaData.DEFAULT_COMPACTION_STRATEGY_CLASS.getSimpleName(); + if (cf_def.compaction_strategy_options == null) + cf_def.compaction_strategy_options = Collections.emptyMap(); + if (!cf_def.isSetCompression_options()) + cf_def.setCompression_options(Collections.singletonMap(CompressionParameters.SSTABLE_COMPRESSION, CFMetaData.DEFAULT_COMPRESSOR)); + if (!cf_def.isSetDefault_time_to_live()) + cf_def.setDefault_time_to_live(CFMetaData.DEFAULT_DEFAULT_TIME_TO_LIVE); + if (!cf_def.isSetDclocal_read_repair_chance()) + cf_def.setDclocal_read_repair_chance(CFMetaData.DEFAULT_DCLOCAL_READ_REPAIR_CHANCE); + + // if index_interval was set, use that for the min_index_interval default + if (!cf_def.isSetMin_index_interval()) + { + if (cf_def.isSetIndex_interval()) + cf_def.setMin_index_interval(cf_def.getIndex_interval()); + else + cf_def.setMin_index_interval(CFMetaData.DEFAULT_MIN_INDEX_INTERVAL); + } + + if (!cf_def.isSetMax_index_interval()) + { + // ensure the max is at least as large as the min + cf_def.setMax_index_interval(Math.max(cf_def.min_index_interval, CFMetaData.DEFAULT_MAX_INDEX_INTERVAL)); + } + } + + /** + * Create CFMetaData from thrift {@link CqlRow} that contains columns from schema_columnfamilies. + * + * @param columnsRes CqlRow containing columns from schema_columnfamilies. + * @return CFMetaData derived from CqlRow + */ + public static CFMetaData fromThriftCqlRow(CqlRow cf, CqlResult columnsRes) + { + UntypedResultSet.Row cfRow = new UntypedResultSet.Row(convertThriftCqlRow(cf)); + + List<Map<String, ByteBuffer>> cols = new ArrayList<>(columnsRes.rows.size()); + for (CqlRow row : columnsRes.rows) + cols.add(convertThriftCqlRow(row)); + UntypedResultSet colsRow = UntypedResultSet.create(cols); + + return CFMetaData.fromSchemaNoTriggers(cfRow, colsRow); + } + + private static Map<String, ByteBuffer> convertThriftCqlRow(CqlRow row) + { + Map<String, ByteBuffer> m = new HashMap<>(); + for (org.apache.cassandra.thrift.Column column : row.getColumns()) + m.put(UTF8Type.instance.getString(column.bufferForName()), column.value); + return m; + } + + public static CfDef toThrift(CFMetaData cfm) + { + CfDef def = new CfDef(cfm.ksName, cfm.cfName); + def.setColumn_type(cfm.cfType.name()); + + if (cfm.isSuper()) + { + def.setComparator_type(cfm.comparator.subtype(0).toString()); + def.setSubcomparator_type(cfm.comparator.subtype(1).toString()); + } + else + { + def.setComparator_type(cfm.comparator.toString()); + } + + def.setComment(Strings.nullToEmpty(cfm.getComment())); + def.setRead_repair_chance(cfm.getReadRepairChance()); + def.setDclocal_read_repair_chance(cfm.getDcLocalReadRepairChance()); + def.setGc_grace_seconds(cfm.getGcGraceSeconds()); + def.setDefault_validation_class(cfm.getDefaultValidator().toString()); + def.setKey_validation_class(cfm.getKeyValidator().toString()); + def.setMin_compaction_threshold(cfm.getMinCompactionThreshold()); + def.setMax_compaction_threshold(cfm.getMaxCompactionThreshold()); + // We only return the alias if only one is set since thrift don't know about multiple key aliases + if (cfm.partitionKeyColumns().size() == 1) + def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes); + def.setColumn_metadata(columnDefinitionsToThrift(cfm.allColumns())); + def.setCompaction_strategy(cfm.compactionStrategyClass.getName()); + def.setCompaction_strategy_options(new HashMap<>(cfm.compactionStrategyOptions)); + def.setCompression_options(cfm.compressionParameters.asThriftOptions()); + def.setBloom_filter_fp_chance(cfm.getBloomFilterFpChance()); + def.setMin_index_interval(cfm.getMinIndexInterval()); + def.setMax_index_interval(cfm.getMaxIndexInterval()); + def.setMemtable_flush_period_in_ms(cfm.getMemtableFlushPeriod()); + def.setCaching(cfm.getCaching().toThriftCaching()); + def.setCells_per_row_to_cache(cfm.getCaching().toThriftCellsPerRow()); + def.setDefault_time_to_live(cfm.getDefaultTimeToLive()); + def.setSpeculative_retry(cfm.getSpeculativeRetry().toString()); + def.setTriggers(triggerDefinitionsToThrift(cfm.getTriggers().values())); + + return def; + } + + public static ColumnDefinition fromThrift(String ksName, + String cfName, + AbstractType<?> thriftComparator, + AbstractType<?> thriftSubcomparator, + ColumnDef thriftColumnDef) + throws SyntaxException, ConfigurationException + { + // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component. + Integer componentIndex = thriftSubcomparator != null ? 1 : null; + AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator; + try + { + comparator.validate(thriftColumnDef.name); + } + catch (MarshalException e) + { + throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator)); + } + + return new ColumnDefinition(ksName, + cfName, + new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator), + TypeParser.parse(thriftColumnDef.validation_class), + thriftColumnDef.index_type == null ? null : org.apache.cassandra.config.IndexType.valueOf(thriftColumnDef.index_type.name()), + thriftColumnDef.index_options, + thriftColumnDef.index_name, + componentIndex, + ColumnDefinition.Kind.REGULAR); + } + + private static List<ColumnDefinition> fromThrift(String ksName, + String cfName, + AbstractType<?> thriftComparator, + AbstractType<?> thriftSubcomparator, + List<ColumnDef> thriftDefs) + throws SyntaxException, ConfigurationException + { + if (thriftDefs == null) + return Collections.emptyList(); + + List<ColumnDefinition> defs = new ArrayList<>(thriftDefs.size()); + for (ColumnDef thriftColumnDef : thriftDefs) + defs.add(fromThrift(ksName, cfName, thriftComparator, thriftSubcomparator, thriftColumnDef)); + + return defs; + } + + @VisibleForTesting + public static ColumnDef toThrift(ColumnDefinition column) + { + ColumnDef cd = new ColumnDef(); + + cd.setName(ByteBufferUtil.clone(column.name.bytes)); + cd.setValidation_class(column.type.toString()); + cd.setIndex_type(column.getIndexType() == null ? null : org.apache.cassandra.thrift.IndexType.valueOf(column.getIndexType().name())); + cd.setIndex_name(column.getIndexName()); + cd.setIndex_options(column.getIndexOptions() == null ? null : Maps.newHashMap(column.getIndexOptions())); + + return cd; + } + + private static List<ColumnDef> columnDefinitionsToThrift(Collection<ColumnDefinition> columns) + { + List<ColumnDef> thriftDefs = new ArrayList<>(columns.size()); + for (ColumnDefinition def : columns) + if (def.kind == ColumnDefinition.Kind.REGULAR) + thriftDefs.add(ThriftConversion.toThrift(def)); + return thriftDefs; + } + + private static Map<String, TriggerDefinition> triggerDefinitionsFromThrift(List<TriggerDef> thriftDefs) + { + Map<String, TriggerDefinition> triggerDefinitions = new HashMap<>(); + for (TriggerDef thriftDef : thriftDefs) + triggerDefinitions.put(thriftDef.getName(), + new TriggerDefinition(thriftDef.getName(), thriftDef.getOptions().get(TriggerDefinition.CLASS))); + return triggerDefinitions; + } + + private static List<TriggerDef> triggerDefinitionsToThrift(Collection<TriggerDefinition> triggers) + { + List<TriggerDef> thriftDefs = new ArrayList<>(triggers.size()); + for (TriggerDefinition def : triggers) + { + TriggerDef td = new TriggerDef(); + td.setName(def.name); + td.setOptions(Collections.singletonMap(TriggerDefinition.CLASS, def.classOption)); + thriftDefs.add(td); + } + return thriftDefs; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/src/java/org/apache/cassandra/tools/BulkLoader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/BulkLoader.java b/src/java/org/apache/cassandra/tools/BulkLoader.java index c639480..be3b810 100644 --- a/src/java/org/apache/cassandra/tools/BulkLoader.java +++ b/src/java/org/apache/cassandra/tools/BulkLoader.java @@ -325,7 +325,7 @@ public class BulkLoader columnFamily); CqlResult columnsRes = client.execute_cql3_query(ByteBufferUtil.bytes(columnsQuery), Compression.NONE, ConsistencyLevel.ONE); - CFMetaData metadata = CFMetaData.fromThriftCqlRow(row, columnsRes); + CFMetaData metadata = ThriftConversion.fromThriftCqlRow(row, columnsRes); knownCfs.put(metadata.cfName, metadata); } break; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/test/unit/org/apache/cassandra/config/CFMetaDataTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java index 2b98da9..79f7f38 100644 --- a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java +++ b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java @@ -36,13 +36,12 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.thrift.CfDef; import org.apache.cassandra.thrift.ColumnDef; import org.apache.cassandra.thrift.IndexType; +import org.apache.cassandra.thrift.ThriftConversion; import org.apache.cassandra.utils.ByteBufferUtil; import org.junit.BeforeClass; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; public class CFMetaDataTest { @@ -82,14 +81,14 @@ public class CFMetaDataTest .setName(CF_STANDARD1); // convert Thrift to CFMetaData - CFMetaData cfMetaData = CFMetaData.fromThrift(cfDef); + CFMetaData cfMetaData = ThriftConversion.fromThrift(cfDef); CfDef thriftCfDef = new CfDef(); thriftCfDef.keyspace = KEYSPACE1; thriftCfDef.name = CF_STANDARD1; thriftCfDef.default_validation_class = cfDef.default_validation_class; thriftCfDef.comment = cfDef.comment; - thriftCfDef.column_metadata = new ArrayList<ColumnDef>(); + thriftCfDef.column_metadata = new ArrayList<>(); for (ColumnDef columnDef : columnDefs) { ColumnDef c = new ColumnDef(); @@ -100,7 +99,7 @@ public class CFMetaDataTest thriftCfDef.column_metadata.add(c); } - CfDef converted = cfMetaData.toThrift(); + CfDef converted = ThriftConversion.toThrift(cfMetaData); assertEquals(thriftCfDef.keyspace, converted.keyspace); assertEquals(thriftCfDef.name, converted.name); @@ -136,7 +135,7 @@ public class CFMetaDataTest // Test thrift conversion CFMetaData before = cfm; - CFMetaData after = CFMetaData.fromThriftForUpdate(before.toThrift(), before); + CFMetaData after = ThriftConversion.fromThriftForUpdate(ThriftConversion.toThrift(before), before); assert before.equals(after) : String.format("%n%s%n!=%n%s", before, after); // Test schema conversion http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java b/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java index 890c46c..2bee0c3 100644 --- a/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java +++ b/test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.thrift.ThriftConversion; import org.apache.cassandra.utils.ByteBufferUtil; public class ColumnDefinitionTest @@ -45,7 +46,7 @@ public class ColumnDefinitionTest protected void testSerializeDeserialize(CFMetaData cfm, ColumnDefinition cd) throws Exception { - ColumnDefinition newCd = ColumnDefinition.fromThrift(cfm.ksName, cfm.cfName, cfm.comparator.asAbstractType(), null, cd.toThrift()); + ColumnDefinition newCd = ThriftConversion.fromThrift(cfm.ksName, cfm.cfName, cfm.comparator.asAbstractType(), null, ThriftConversion.toThrift(cd)); Assert.assertNotSame(cd, newCd); Assert.assertEquals(cd.hashCode(), newCd.hashCode()); Assert.assertEquals(cd, newCd); http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java index f6d4ad4..edfd7b6 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java @@ -29,6 +29,7 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.SimpleStrategy; import org.apache.cassandra.service.MigrationManager; +import org.apache.cassandra.thrift.ThriftConversion; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -45,7 +46,7 @@ public class DatabaseDescriptorTest { for (CFMetaData cfm : Schema.instance.getKeyspaceMetaData(keyspaceName).values()) { - CFMetaData cfmDupe = CFMetaData.fromThrift(cfm.toThrift()); + CFMetaData cfmDupe = ThriftConversion.fromThrift(ThriftConversion.toThrift(cfm)); assertNotNull(cfmDupe); assertEquals(cfm, cfmDupe); } @@ -58,7 +59,7 @@ public class DatabaseDescriptorTest for (KSMetaData ksm : Schema.instance.getKeyspaceDefinitions()) { // Not testing round-trip on the KsDef via serDe() because maps - KSMetaData ksmDupe = KSMetaData.fromThrift(ksm.toThrift()); + KSMetaData ksmDupe = ThriftConversion.fromThrift(ThriftConversion.toThrift(ksm)); assertNotNull(ksmDupe); assertEquals(ksm, ksmDupe); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a94b173e/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java index e81dd3d..708c29a 100644 --- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java +++ b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java @@ -161,7 +161,7 @@ public class ThriftValidationTest try { - KSMetaData.fromThrift(ks_def).validate(); + ThriftConversion.fromThrift(ks_def).validate(); } catch (ConfigurationException e) { @@ -176,7 +176,7 @@ public class ThriftValidationTest try { - KSMetaData.fromThrift(ks_def).validate(); + ThriftConversion.fromThrift(ks_def).validate(); } catch (ConfigurationException e) { @@ -191,7 +191,7 @@ public class ThriftValidationTest try { - KSMetaData.fromThrift(ks_def).validate(); + ThriftConversion.fromThrift(ks_def).validate(); } catch (ConfigurationException e) {
