Repository: cassandra Updated Branches: refs/heads/trunk 2306fb877 -> 99ba549d0
Reduce run time for CQL tests patch by slebresne; reviewed by thobbs for CASSANDRA-7327 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/78e91c4c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/78e91c4c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/78e91c4c Branch: refs/heads/trunk Commit: 78e91c4cacb7128a994a7a8ca16f5a9e20b576ce Parents: 285c369 Author: Sylvain Lebresne <sylv...@datastax.com> Authored: Thu May 29 16:16:53 2014 +0200 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Thu Jun 5 08:39:48 2014 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + build.xml | 9 ++- src/java/org/apache/cassandra/auth/Auth.java | 2 +- .../cql3/statements/AlterKeyspaceStatement.java | 4 +- .../cql3/statements/AlterTableStatement.java | 4 +- .../cql3/statements/AlterTypeStatement.java | 8 +- .../cql3/statements/CreateIndexStatement.java | 4 +- .../statements/CreateKeyspaceStatement.java | 4 +- .../cql3/statements/CreateTableStatement.java | 4 +- .../cql3/statements/CreateTriggerStatement.java | 4 +- .../cql3/statements/CreateTypeStatement.java | 4 +- .../cql3/statements/DropIndexStatement.java | 4 +- .../cql3/statements/DropKeyspaceStatement.java | 4 +- .../cql3/statements/DropTableStatement.java | 4 +- .../cql3/statements/DropTriggerStatement.java | 4 +- .../cql3/statements/DropTypeStatement.java | 4 +- .../statements/SchemaAlteringStatement.java | 16 +++- .../org/apache/cassandra/db/DefsTables.java | 10 ++- src/java/org/apache/cassandra/db/Memtable.java | 7 +- .../cassandra/service/MigrationManager.java | 85 +++++++++++++++++--- .../cassandra/service/StorageService.java | 2 +- .../unit/org/apache/cassandra/SchemaLoader.java | 7 +- .../org/apache/cassandra/cql3/CQLTester.java | 74 +++++++++++++++-- 23 files changed, 202 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 23f36c8..41ee6e5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Make sure we clear out repair sessions from netstats (CASSANDRA-7329) * Don't fail streams on failure detector downs (CASSANDRA-3569) * Add optional keyspace to DROP INDEX statement (CASSANDRA-7314) + * Reduce run time for CQL tests (CASSANDRA-7327) Merged from 2.0: * Add per-CF range read request latency metrics (CASSANDRA-7338) * Fix NPE in StreamTransferTask.createMessageForRetry() (CASSANDRA-7323) http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 9f373c4..7e34904 100644 --- a/build.xml +++ b/build.xml @@ -1078,6 +1078,8 @@ <jvmarg value="-javaagent:${basedir}/lib/jamm-0.2.6.jar" /> <jvmarg value="-ea"/> <jvmarg value="-Xss256k"/> + <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/> + <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/> <optjvmargs/> <classpath> <path refid="cassandra.classpath" /> @@ -1206,8 +1208,8 @@ <target name="cql-test" depends="build-test" description="Execute CQL tests"> <testmacro suitename="cql" inputdir="${test.unit.src}" - timeout="${test.timeout}" filter="**/cql3/*Test.java"> - <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/> + timeout="${test.timeout}" filter="**/cql3/*Test.java" + forkmode="once"> </testmacro> </target> @@ -1216,7 +1218,7 @@ <echo message="running ${test.methods} tests from ${test.name}"/> <mkdir dir="${build.test.dir}/cassandra"/> <mkdir dir="${build.test.dir}/output"/> - <junit fork="on" failureproperty="testfailed" maxmemory="1024m" timeout="${test.timeout}"> + <junit fork="on" forkmode="once" failureproperty="testfailed" maxmemory="1024m" timeout="${test.timeout}"> <sysproperty key="net.sourceforge.cobertura.datafile" file="${cobertura.datafile}"/> <formatter type="brief" usefile="false"/> <jvmarg value="-Dstorage-config=${test.conf}"/> @@ -1225,6 +1227,7 @@ <jvmarg value="-ea"/> <jvmarg value="-Xss256k"/> <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/> + <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/> <classpath> <path refid="cassandra.classpath" /> <pathelement location="${test.classes}"/> http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/auth/Auth.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/auth/Auth.java b/src/java/org/apache/cassandra/auth/Auth.java index 587fc66..7c532b0 100644 --- a/src/java/org/apache/cassandra/auth/Auth.java +++ b/src/java/org/apache/cassandra/auth/Auth.java @@ -181,7 +181,7 @@ public class Auth try { KSMetaData ksm = KSMetaData.newKeyspace(AUTH_KS, SimpleStrategy.class.getName(), ImmutableMap.of("replication_factor", "1"), true); - MigrationManager.announceNewKeyspace(ksm, 0); + MigrationManager.announceNewKeyspace(ksm, 0, false); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java index 39d1cde..a940f9e 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java @@ -79,14 +79,14 @@ public class AlterKeyspaceStatement extends SchemaAlteringStatement } } - public void announceMigration() throws RequestValidationException + public void announceMigration(boolean isLocalOnly) throws RequestValidationException { KSMetaData ksm = Schema.instance.getKSMetaData(name); // In the (very) unlikely case the keyspace was dropped since validate() if (ksm == null) throw new InvalidRequestException("Unknown keyspace " + name); - MigrationManager.announceKeyspaceUpdate(attrs.asKSMetadataUpdate(ksm)); + MigrationManager.announceKeyspaceUpdate(attrs.asKSMetadataUpdate(ksm), isLocalOnly); } public ResultMessage.SchemaChange.Change changeType() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java index 27acb3c..cff0e8f 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java @@ -74,7 +74,7 @@ public class AlterTableStatement extends SchemaAlteringStatement // validated in announceMigration() } - public void announceMigration() throws RequestValidationException + public void announceMigration(boolean isLocalOnly) throws RequestValidationException { CFMetaData meta = validateColumnFamily(keyspace(), columnFamily()); CFMetaData cfm = meta.copy(); @@ -237,7 +237,7 @@ public class AlterTableStatement extends SchemaAlteringStatement break; } - MigrationManager.announceColumnFamilyUpdate(cfm, false); + MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly); } public String toString() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java index 1996457..3cac7cc 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java @@ -89,7 +89,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement return name.getKeyspace(); } - public void announceMigration() throws InvalidRequestException, ConfigurationException + public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException { KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace()); if (ksm == null) @@ -104,7 +104,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement // Now, we need to announce the type update to basically change it for new tables using this type, // but we also need to find all existing user types and CF using it and change them. - MigrationManager.announceTypeUpdate(updated); + MigrationManager.announceTypeUpdate(updated, isLocalOnly); for (KSMetaData ksm2 : Schema.instance.getKeyspaceDefinitions()) { @@ -115,7 +115,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement for (ColumnDefinition def : copy.allColumns()) modified |= updateDefinition(copy, def, toUpdate.keyspace, toUpdate.name, updated); if (modified) - MigrationManager.announceColumnFamilyUpdate(copy, false); + MigrationManager.announceColumnFamilyUpdate(copy, false, isLocalOnly); } // Other user types potentially using the updated type @@ -131,7 +131,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement } AbstractType<?> upd = updateWith(ut, toUpdate.keyspace, toUpdate.name, updated); if (upd != null) - MigrationManager.announceTypeUpdate((UserType)upd); + MigrationManager.announceTypeUpdate((UserType)upd, isLocalOnly); } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java index ea1f933..b2e8d59 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java @@ -117,7 +117,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement throw new InvalidRequestException(String.format("Cannot add secondary index to already primarily indexed column %s", target.column)); } - public void announceMigration() throws RequestValidationException + public void announceMigration(boolean isLocalOnly) throws RequestValidationException { logger.debug("Updating column {} definition for index {}", target.column, indexName); CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy(); @@ -147,7 +147,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement cd.setIndexName(indexName); cfm.addDefaultIndexNames(); - MigrationManager.announceColumnFamilyUpdate(cfm, false); + MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly); } public ResultMessage.SchemaChange.Change changeType() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java index 2ed1d91..c528e16 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java @@ -97,11 +97,11 @@ public class CreateKeyspaceStatement extends SchemaAlteringStatement attrs.getReplicationOptions()); } - public void announceMigration() throws RequestValidationException + public void announceMigration(boolean isLocalOnly) throws RequestValidationException { try { - MigrationManager.announceNewKeyspace(attrs.asKSMetadata(name)); + MigrationManager.announceNewKeyspace(attrs.asKSMetadata(name), isLocalOnly); } catch (AlreadyExistsException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java index 7cef999..d764845 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java @@ -105,11 +105,11 @@ public class CreateTableStatement extends SchemaAlteringStatement return columnDefs; } - public void announceMigration() throws RequestValidationException + public void announceMigration(boolean isLocalOnly) throws RequestValidationException { try { - MigrationManager.announceNewColumnFamily(getCFMetaData()); + MigrationManager.announceNewColumnFamily(getCFMetaData(), isLocalOnly); } catch (AlreadyExistsException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java index 0e14d2e..e99da8a 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java @@ -65,12 +65,12 @@ public class CreateTriggerStatement extends SchemaAlteringStatement } } - public void announceMigration() throws ConfigurationException + public void announceMigration(boolean isLocalOnly) throws ConfigurationException { CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy(); cfm.addTriggerDefinition(TriggerDefinition.create(triggerName, triggerClass)); logger.info("Adding trigger with name {} and class {}", triggerName, triggerClass); - MigrationManager.announceColumnFamilyUpdate(cfm, false); + MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly); } public ResultMessage.SchemaChange.Change changeType() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java index cc5447e..41116f5 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java @@ -111,7 +111,7 @@ public class CreateTypeStatement extends SchemaAlteringStatement return new UserType(name.getKeyspace(), name.getUserTypeName(), names, types); } - public void announceMigration() throws InvalidRequestException, ConfigurationException + public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException { KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace()); assert ksm != null; // should haven't validate otherwise @@ -122,6 +122,6 @@ public class CreateTypeStatement extends SchemaAlteringStatement UserType type = createType(); checkForDuplicateNames(type); - MigrationManager.announceNewType(type); + MigrationManager.announceNewType(type, isLocalOnly); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java index 44704a9..4321886 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java @@ -60,14 +60,14 @@ public class DropIndexStatement extends SchemaAlteringStatement return ResultMessage.SchemaChange.Change.UPDATED; } - public void announceMigration() throws InvalidRequestException, ConfigurationException + public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException { CFMetaData cfm = findIndexedCF(); if (cfm == null) return; CFMetaData updatedCfm = updateCFMetadata(cfm); - MigrationManager.announceColumnFamilyUpdate(updatedCfm, false); + MigrationManager.announceColumnFamilyUpdate(updatedCfm, false, isLocalOnly); } private CFMetaData updateCFMetadata(CFMetaData cfm) http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java index 30fd964..6c99bdd 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java @@ -55,11 +55,11 @@ public class DropKeyspaceStatement extends SchemaAlteringStatement return keyspace; } - public void announceMigration() throws ConfigurationException + public void announceMigration(boolean isLocalOnly) throws ConfigurationException { try { - MigrationManager.announceKeyspaceDrop(keyspace); + MigrationManager.announceKeyspaceDrop(keyspace, isLocalOnly); } catch(ConfigurationException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java index d27261c..443c6d3 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java @@ -54,11 +54,11 @@ public class DropTableStatement extends SchemaAlteringStatement // validated in announceMigration() } - public void announceMigration() throws ConfigurationException + public void announceMigration(boolean isLocalOnly) throws ConfigurationException { try { - MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily()); + MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily(), isLocalOnly); } catch (ConfigurationException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java index c9b33a1..ff7948f 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java @@ -53,13 +53,13 @@ public class DropTriggerStatement extends SchemaAlteringStatement ThriftValidation.validateColumnFamily(keyspace(), columnFamily()); } - public void announceMigration() throws ConfigurationException + public void announceMigration(boolean isLocalOnly) throws ConfigurationException { CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy(); if (!cfm.removeTrigger(triggerName)) throw new ConfigurationException(String.format("Trigger %s was not found", triggerName)); logger.info("Dropping trigger with name {}", triggerName); - MigrationManager.announceColumnFamilyUpdate(cfm, false); + MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly); } public ResultMessage.SchemaChange.Change changeType() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java index 1e1ded5..4036d33 100644 --- a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java @@ -137,7 +137,7 @@ public class DropTypeStatement extends SchemaAlteringStatement return name.getKeyspace(); } - public void announceMigration() throws InvalidRequestException, ConfigurationException + public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException { KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace()); assert ksm != null; @@ -145,6 +145,6 @@ public class DropTypeStatement extends SchemaAlteringStatement UserType toDrop = ksm.userTypes.getType(name.getUserTypeName()); // Can be null with ifExists if (toDrop != null) - MigrationManager.announceTypeDrop(toDrop); + MigrationManager.announceTypeDrop(toDrop, isLocalOnly); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java b/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java index 94df854..7d454a5 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java @@ -64,18 +64,26 @@ public abstract class SchemaAlteringStatement extends CFStatement implements CQL public abstract ResultMessage.SchemaChange.Change changeType(); - public abstract void announceMigration() throws RequestValidationException; + public abstract void announceMigration(boolean isLocalOnly) throws RequestValidationException; public ResultMessage execute(QueryState state, QueryOptions options) throws RequestValidationException { - announceMigration(); + announceMigration(false); String tableName = cfName == null || columnFamily() == null ? "" : columnFamily(); return new ResultMessage.SchemaChange(changeType(), keyspace(), tableName); } public ResultMessage executeInternal(QueryState state, QueryOptions options) { - // executeInternal is for local query only, thus altering schema is not supported - throw new UnsupportedOperationException(); + try + { + announceMigration(true); + String tableName = cfName == null || columnFamily() == null ? "" : columnFamily(); + return new ResultMessage.SchemaChange(changeType(), keyspace(), tableName); + } + catch (RequestValidationException e) + { + throw new RuntimeException(e); + } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/db/DefsTables.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/DefsTables.java b/src/java/org/apache/cassandra/db/DefsTables.java index 3902e05..ede3ebd 100644 --- a/src/java/org/apache/cassandra/db/DefsTables.java +++ b/src/java/org/apache/cassandra/db/DefsTables.java @@ -161,6 +161,12 @@ public class DefsTables */ public static synchronized void mergeSchema(Collection<Mutation> mutations) throws ConfigurationException, IOException { + mergeSchemaInternal(mutations, true); + Schema.instance.updateVersionAndAnnounce(); + } + + public static synchronized void mergeSchemaInternal(Collection<Mutation> mutations, boolean doFlush) throws ConfigurationException, IOException + { // current state of the schema Map<DecoratedKey, ColumnFamily> oldKeyspaces = SystemKeyspace.getSchema(SystemKeyspace.SCHEMA_KEYSPACES_CF); Map<DecoratedKey, ColumnFamily> oldColumnFamilies = SystemKeyspace.getSchema(SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF); @@ -169,7 +175,7 @@ public class DefsTables for (Mutation mutation : mutations) mutation.apply(); - if (!StorageService.instance.isClientMode()) + if (doFlush && !StorageService.instance.isClientMode()) flushSchemaCFs(); // with new data applied @@ -184,8 +190,6 @@ public class DefsTables // it is safe to drop a keyspace only when all nested ColumnFamilies where deleted for (String keyspaceToDrop : keyspacesToDrop) dropKeyspace(keyspaceToDrop); - - Schema.instance.updateVersionAndAnnounce(); } private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated) http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/db/Memtable.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java index 417b4a7..3c02678 100644 --- a/src/java/org/apache/cassandra/db/Memtable.java +++ b/src/java/org/apache/cassandra/db/Memtable.java @@ -53,7 +53,7 @@ public class Memtable private static final Logger logger = LoggerFactory.getLogger(Memtable.class); static final MemtablePool MEMORY_POOL = DatabaseDescriptor.getMemtableAllocatorPool(); - private static final int ROW_OVERHEAD_HEAP_SIZE; + private static final int ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(Integer.valueOf(System.getProperty("cassandra.memtable_row_overhead_computation_step", "100000"))); private final MemtableAllocator allocator; private final AtomicLong liveDataSize = new AtomicLong(0); @@ -389,14 +389,13 @@ public class Memtable } } - static + private static int estimateRowOverhead(final int count) { // calculate row overhead final OpOrder.Group group = new OpOrder().start(); int rowOverhead; MemtableAllocator allocator = MEMORY_POOL.newAllocator(); ConcurrentNavigableMap<RowPosition, Object> rows = new ConcurrentSkipListMap<>(); - final int count = 100000; final Object val = new Object(); for (int i = 0 ; i < count ; i++) rows.put(allocator.clone(new BufferDecoratedKey(new LongToken((long) i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val); @@ -406,6 +405,6 @@ public class Memtable rowOverhead += AtomicBTreeColumns.EMPTY_SIZE; allocator.setDiscarding(); allocator.setDiscarded(); - ROW_OVERHEAD_HEAP_SIZE = rowOverhead; + return rowOverhead; } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/service/MigrationManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/MigrationManager.java b/src/java/org/apache/cassandra/service/MigrationManager.java index ec46d3f..a6408ec 100644 --- a/src/java/org/apache/cassandra/service/MigrationManager.java +++ b/src/java/org/apache/cassandra/service/MigrationManager.java @@ -211,10 +211,15 @@ public class MigrationManager public static void announceNewKeyspace(KSMetaData ksm) throws ConfigurationException { - announceNewKeyspace(ksm, FBUtilities.timestampMicros()); + announceNewKeyspace(ksm, false); } - public static void announceNewKeyspace(KSMetaData ksm, long timestamp) throws ConfigurationException + public static void announceNewKeyspace(KSMetaData ksm, boolean announceLocally) throws ConfigurationException + { + announceNewKeyspace(ksm, FBUtilities.timestampMicros(), announceLocally); + } + + public static void announceNewKeyspace(KSMetaData ksm, long timestamp, boolean announceLocally) throws ConfigurationException { ksm.validate(); @@ -222,11 +227,16 @@ public class MigrationManager throw new AlreadyExistsException(ksm.name); logger.info(String.format("Create new Keyspace: %s", ksm)); - announce(ksm.toSchema(timestamp)); + announce(ksm.toSchema(timestamp), announceLocally); } public static void announceNewColumnFamily(CFMetaData cfm) throws ConfigurationException { + announceNewColumnFamily(cfm, false); + } + + public static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally) throws ConfigurationException + { cfm.validate(); KSMetaData ksm = Schema.instance.getKSMetaData(cfm.ksName); @@ -236,16 +246,26 @@ public class MigrationManager throw new AlreadyExistsException(cfm.ksName, cfm.cfName); logger.info(String.format("Create new ColumnFamily: %s", cfm)); - announce(addSerializedKeyspace(cfm.toSchema(FBUtilities.timestampMicros()), cfm.ksName)); + announce(addSerializedKeyspace(cfm.toSchema(FBUtilities.timestampMicros()), cfm.ksName), announceLocally); } public static void announceNewType(UserType newType) { - announce(UTMetaData.toSchema(newType, FBUtilities.timestampMicros())); + announceNewType(newType, false); + } + + public static void announceNewType(UserType newType, boolean announceLocally) + { + announce(UTMetaData.toSchema(newType, FBUtilities.timestampMicros()), announceLocally); } public static void announceKeyspaceUpdate(KSMetaData ksm) throws ConfigurationException { + announceKeyspaceUpdate(ksm, false); + } + + public static void announceKeyspaceUpdate(KSMetaData ksm, boolean announceLocally) throws ConfigurationException + { ksm.validate(); KSMetaData oldKsm = Schema.instance.getKSMetaData(ksm.name); @@ -253,11 +273,16 @@ public class MigrationManager throw new ConfigurationException(String.format("Cannot update non existing keyspace '%s'.", ksm.name)); logger.info(String.format("Update Keyspace '%s' From %s To %s", ksm.name, oldKsm, ksm)); - announce(oldKsm.toSchemaUpdate(ksm, FBUtilities.timestampMicros())); + announce(oldKsm.toSchemaUpdate(ksm, FBUtilities.timestampMicros()), announceLocally); } public static void announceColumnFamilyUpdate(CFMetaData cfm, boolean fromThrift) throws ConfigurationException { + announceColumnFamilyUpdate(cfm, fromThrift, false); + } + + public static void announceColumnFamilyUpdate(CFMetaData cfm, boolean fromThrift, boolean announceLocally) throws ConfigurationException + { cfm.validate(); CFMetaData oldCfm = Schema.instance.getCFMetaData(cfm.ksName, cfm.cfName); @@ -267,32 +292,47 @@ public class MigrationManager oldCfm.validateCompatility(cfm); logger.info(String.format("Update ColumnFamily '%s/%s' From %s To %s", cfm.ksName, cfm.cfName, oldCfm, cfm)); - announce(addSerializedKeyspace(oldCfm.toSchemaUpdate(cfm, FBUtilities.timestampMicros(), fromThrift), cfm.ksName)); + announce(addSerializedKeyspace(oldCfm.toSchemaUpdate(cfm, FBUtilities.timestampMicros(), fromThrift), cfm.ksName), announceLocally); } public static void announceTypeUpdate(UserType updatedType) { - announceNewType(updatedType); + announceTypeUpdate(updatedType, false); + } + + public static void announceTypeUpdate(UserType updatedType, boolean announceLocally) + { + announceNewType(updatedType, announceLocally); } public static void announceKeyspaceDrop(String ksName) throws ConfigurationException { + announceKeyspaceDrop(ksName, false); + } + + public static void announceKeyspaceDrop(String ksName, boolean announceLocally) throws ConfigurationException + { KSMetaData oldKsm = Schema.instance.getKSMetaData(ksName); if (oldKsm == null) throw new ConfigurationException(String.format("Cannot drop non existing keyspace '%s'.", ksName)); logger.info(String.format("Drop Keyspace '%s'", oldKsm.name)); - announce(oldKsm.dropFromSchema(FBUtilities.timestampMicros())); + announce(oldKsm.dropFromSchema(FBUtilities.timestampMicros()), announceLocally); } public static void announceColumnFamilyDrop(String ksName, String cfName) throws ConfigurationException { + announceColumnFamilyDrop(ksName, cfName, false); + } + + public static void announceColumnFamilyDrop(String ksName, String cfName, boolean announceLocally) throws ConfigurationException + { CFMetaData oldCfm = Schema.instance.getCFMetaData(ksName, cfName); if (oldCfm == null) throw new ConfigurationException(String.format("Cannot drop non existing column family '%s' in keyspace '%s'.", cfName, ksName)); logger.info(String.format("Drop ColumnFamily '%s/%s'", oldCfm.ksName, oldCfm.cfName)); - announce(addSerializedKeyspace(oldCfm.dropFromSchema(FBUtilities.timestampMicros()), ksName)); + announce(addSerializedKeyspace(oldCfm.dropFromSchema(FBUtilities.timestampMicros()), ksName), announceLocally); } // Include the serialized keyspace for when a target node missed the CREATE KEYSPACE migration (see #5631). @@ -304,16 +344,35 @@ public class MigrationManager public static void announceTypeDrop(UserType droppedType) { - announce(UTMetaData.dropFromSchema(droppedType, FBUtilities.timestampMicros())); + announceTypeDrop(droppedType, false); + } + + public static void announceTypeDrop(UserType droppedType, boolean announceLocally) + { + announce(UTMetaData.dropFromSchema(droppedType, FBUtilities.timestampMicros()), announceLocally); } /** * actively announce a new version to active hosts via rpc * @param schema The schema mutation to be applied */ - private static void announce(Mutation schema) + private static void announce(Mutation schema, boolean announceLocally) { - FBUtilities.waitOnFuture(announce(Collections.singletonList(schema))); + if (announceLocally) + { + try + { + DefsTables.mergeSchemaInternal(Collections.singletonList(schema), false); + } + catch (ConfigurationException | IOException e) + { + throw new RuntimeException(e); + } + } + else + { + FBUtilities.waitOnFuture(announce(Collections.singletonList(schema))); + } } private static void pushSchemaMutation(InetAddress endpoint, Collection<Mutation> schema) http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index a363279..90f43ac 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -890,7 +890,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (Schema.instance.getKSMetaData(Tracing.TRACE_KS) == null) { KSMetaData tracingKeyspace = KSMetaData.traceKeyspace(); - MigrationManager.announceNewKeyspace(tracingKeyspace, 0); + MigrationManager.announceNewKeyspace(tracingKeyspace, 0, false); } if (!isSurveyMode) http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/test/unit/org/apache/cassandra/SchemaLoader.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java index 7f637a4..a9d69fd 100644 --- a/test/unit/org/apache/cassandra/SchemaLoader.java +++ b/test/unit/org/apache/cassandra/SchemaLoader.java @@ -54,6 +54,10 @@ public class SchemaLoader { prepareServer(); + // Migrations aren't happy if gossiper is not started. Even if we don't use migrations though, + // some tests now expect us to start gossip for them. + startGossiper(); + // if you're messing with low-level sstable stuff, it can be useful to inject the schema directly // Schema.instance.load(schemaDefinition()); for (KSMetaData ksm : schemaDefinition()) @@ -76,9 +80,6 @@ public class SchemaLoader }); Keyspace.setInitialized(); - // Migrations aren't happy if gossiper is not started. Even if we don't use migrations though, - // some tests now expect us to start gossip for them. - startGossiper(); } public static void startGossiper() http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/test/unit/org/apache/cassandra/cql3/CQLTester.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 06119ed..b8912e7 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -17,12 +17,16 @@ */ package org.apache.cassandra.cql3; +import java.io.File; import java.nio.ByteBuffer; import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.base.Objects; import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.slf4j.Logger; @@ -30,8 +34,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; /** @@ -42,26 +49,79 @@ public abstract class CQLTester protected static final Logger logger = LoggerFactory.getLogger(CQLTester.class); private static final String KEYSPACE = "cql_test_keyspace"; - private static final boolean USE_PREPARED_VALUES = Boolean.valueOf(System.getProperty("cassandra.test.use_prepared", "true")); - private static final AtomicInteger seqNumber = new AtomicInteger(); + static + { + // Once per-JVM is enough + SchemaLoader.prepareServer(); + } + private String currentTable; @BeforeClass public static void setUpClass() throws Throwable { - // This start gossiper for the sake of schema migrations. We might be able to get rid of that with some work. - SchemaLoader.prepareServer(); - schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", KEYSPACE)); } @AfterClass public static void tearDownClass() { - SchemaLoader.stopGossiper(); + } + + @After + public void afterTest() throws Throwable + { + if (currentTable == null) + return; + + final String toDrop = currentTable; + currentTable = null; + + // We want to clean up after the test, but dropping a table is rather long so just do that asynchronously + StorageService.tasks.execute(new Runnable() + { + public void run() + { + try + { + schemaChange(String.format("DROP TABLE %s.%s", KEYSPACE, toDrop)); + + // Dropping doesn't delete the sstables. It's not a huge deal but it's cleaner to cleanup after us + // Thas said, we shouldn't delete blindly before the SSTableDeletingTask for the table we drop + // have run or they will be unhappy. Since those taks are scheduled on StorageService.tasks and that's + // mono-threaded, just push a task on the queue to find when it's empty. No perfect but good enough. + + final CountDownLatch latch = new CountDownLatch(1); + StorageService.tasks.execute(new Runnable() + { + public void run() + { + latch.countDown(); + } + }); + latch.await(2, TimeUnit.SECONDS); + + removeAllSSTables(KEYSPACE, toDrop); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + + private static void removeAllSSTables(String ks, String table) + { + // clean up data directory which are stored as data directory/keyspace/data files + for (File d : Directories.getKSChildDirectories(ks)) + { + if (d.exists() && d.getName().contains(table)) + FileUtils.deleteRecursive(d); + } } protected void createTable(String query) @@ -77,7 +137,7 @@ public abstract class CQLTester try { // executeOnceInternal don't work for schema changes - QueryProcessor.process(query, ConsistencyLevel.ONE); + QueryProcessor.executeOnceInternal(query); } catch (Exception e) {
