Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/fb92a526 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/fb92a526 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/fb92a526 Branch: refs/heads/trunk Commit: fb92a52624c13969386760b967b7168a87e857bb Parents: f4014bb 4e834c5 Author: Marcus Eriksson <marc...@apache.org> Authored: Tue Sep 5 10:38:39 2017 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Tue Sep 5 10:38:39 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../io/sstable/format/SSTableReader.java | 8 +- .../sstable/IndexSummaryRedistributionTest.java | 142 +++++++++++++++++++ 3 files changed, 147 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 870351c,1abd7de..1ed3fe6 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,39 -1,9 +1,40 @@@ -2.2.11 +3.0.15 + * Fix MV timestamp issues (CASSANDRA-11500) + * Better tolerate improperly formatted bcrypt hashes (CASSANDRA-13626) + * Fix race condition in read command serialization (CASSANDRA-13363) + * Enable segement creation before recovering commitlogs (CASSANDRA-13587) + * Fix AssertionError in short read protection (CASSANDRA-13747) + * Don't skip corrupted sstables on startup (CASSANDRA-13620) + * Fix the merging of cells with different user type versions (CASSANDRA-13776) + * Copy session properties on cqlsh.py do_login (CASSANDRA-13640) + * Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719) + * Don't let stress write warmup data if n=0 (CASSANDRA-13773) + * Gossip thread slows down when using batch commit log (CASSANDRA-12966) + * Randomize batchlog endpoint selection with only 1 or 2 racks (CASSANDRA-12884) + * Fix digest calculation for counter cells (CASSANDRA-13750) + * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573) + * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737) + * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730) + * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371) + * Fix incorrect [2.1 <- 3.0] serialization of counter cells created in 2.0 (CASSANDRA-13691) + * Fix invalid writetime for null cells (CASSANDRA-13711) + * Fix ALTER TABLE statement to atomically propagate changes to the table and its MVs (CASSANDRA-12952) + * Fixed ambiguous output of nodetool tablestats command (CASSANDRA-13722) + * JMXEnabledThreadPoolExecutor with corePoolSize equal to maxPoolSize (Backport CASSANDRA-13329) + * Fix Digest mismatch Exception if hints file has UnknownColumnFamily (CASSANDRA-13696) + * Purge tombstones created by expired cells (CASSANDRA-13643) + * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482) + * Set test.runners based on cores and memory size (CASSANDRA-13078) + * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557) + * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606) + * Fix secondary index queries on COMPACT tables (CASSANDRA-13627) + * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568) + * sstabledump reports incorrect usage for argument order (CASSANDRA-13532) +Merged from 2.2: + * Fix load over calculated issue in IndexSummaryRedistribution (CASSANDRA-13738) * Fix compaction and flush exception not captured (CASSANDRA-13833) - * Make BatchlogManagerMBean.forceBatchlogReplay() blocking (CASSANDRA-13809) * Uncaught exceptions in Netty pipeline (CASSANDRA-13649) - * Prevent integer overflow on exabyte filesystems (CASSANDRA-13067) + * Prevent integer overflow on exabyte filesystems (CASSANDRA-13067) * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223) * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272) * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592) http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index d56b3e7,6666885..25ba212 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@@ -1123,10 -1169,14 +1123,10 @@@ public abstract class SSTableReader ext saveSummary(ibuilder, dbuilder, newSummary); } - long newSize = bytesOnDisk(); - StorageMetrics.load.inc(newSize - oldSize); - parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize); - parent.metric.totalDiskSpaceUsed.inc(newSize - oldSize); + // The new size will be added in Transactional.commit() as an updated SSTable, more details: CASSANDRA-13738 + StorageMetrics.load.dec(oldSize); + parent.metric.liveDiskSpaceUsed.dec(oldSize); ++ parent.metric.totalDiskSpaceUsed.dec(oldSize); return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java index 0000000,77fd69a..31a57e1 mode 000000,100644..100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java @@@ -1,0 -1,145 +1,142 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.cassandra.io.sstable; + + import java.io.IOException; + import java.nio.ByteBuffer; + import java.util.ArrayList; + import java.util.List; + import java.util.concurrent.ExecutionException; + import java.util.concurrent.Future; + + import org.junit.BeforeClass; + import org.junit.Test; + + import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.cache.CachingOptions; -import org.apache.cassandra.config.KSMetaData; + import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; + import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Mutation; ++import org.apache.cassandra.db.RowUpdateBuilder; + import org.apache.cassandra.exceptions.ConfigurationException; + import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.SimpleStrategy; + import org.apache.cassandra.metrics.RestorableMeter; + import org.apache.cassandra.metrics.StorageMetrics; ++import org.apache.cassandra.schema.CachingParams; ++import org.apache.cassandra.schema.KeyspaceParams; + + import static org.junit.Assert.assertEquals; + + public class IndexSummaryRedistributionTest + { + private static final String KEYSPACE1 = "IndexSummaryRedistributionTest"; + private static final String CF_STANDARD = "Standard"; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), ++ KeyspaceParams.simple(1), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD) + .minIndexInterval(8) + .maxIndexInterval(256) - .caching(CachingOptions.NONE)); ++ .caching(CachingParams.CACHE_NOTHING)); + } + + @Test + public void testMetricsLoadAfterRedistribution() throws IOException + { + String ksname = KEYSPACE1; + String cfname = CF_STANDARD; + Keyspace keyspace = Keyspace.open(ksname); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); + int numSSTables = 1; + int numRows = 1024 * 10; + long load = StorageMetrics.load.getCount(); + StorageMetrics.load.dec(load); // reset the load metric + createSSTables(ksname, cfname, numSSTables, numRows); + - List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables()); ++ List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables()); + for (SSTableReader sstable : sstables) + sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); + + long oldSize = 0; + for (SSTableReader sstable : sstables) + { - assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001); ++ assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); + oldSize += sstable.bytesOnDisk(); + } + + load = StorageMetrics.load.getCount(); - + long others = load - oldSize; // Other SSTables size, e.g. schema and other system SSTables + - int originalMinIndexInterval = cfs.metadata.getMinIndexInterval(); ++ int originalMinIndexInterval = cfs.metadata.params.minIndexInterval; + // double the min_index_interval + cfs.metadata.minIndexInterval(originalMinIndexInterval * 2); + IndexSummaryManager.instance.redistributeSummaries(); + + long newSize = 0; - for (SSTableReader sstable : cfs.getSSTables()) ++ for (SSTableReader sstable : cfs.getLiveSSTables()) + { - assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / cfs.metadata.getMinIndexInterval(), sstable.getIndexSummarySize()); ++ assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); ++ assertEquals(numRows / cfs.metadata.params.minIndexInterval, sstable.getIndexSummarySize()); + newSize += sstable.bytesOnDisk(); + } + newSize += others; + load = StorageMetrics.load.getCount(); + + // new size we calculate should be almost the same as the load in metrics + assertEquals(newSize, load, newSize / 10); + } + + private void createSSTables(String ksname, String cfname, int numSSTables, int numRows) + { + Keyspace keyspace = Keyspace.open(ksname); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); + cfs.truncateBlocking(); + cfs.disableAutoCompaction(); + + ArrayList<Future> futures = new ArrayList<>(numSSTables); + ByteBuffer value = ByteBuffer.wrap(new byte[100]); + for (int sstable = 0; sstable < numSSTables; sstable++) + { + for (int row = 0; row < numRows; row++) + { - DecoratedKey key = Util.dk(String.format("%3d", row)); - Mutation rm = new Mutation(ksname, key.getKey()); - rm.add(cfname, Util.cellname("column"), value, 0); - rm.applyUnsafe(); ++ String key = String.format("%3d", row); ++ new RowUpdateBuilder(cfs.metadata, 0, key) ++ .clustering("column") ++ .add("val", value) ++ .build() ++ .applyUnsafe(); + } + futures.add(cfs.forceFlush()); + } + for (Future future : futures) + { + try + { + future.get(); + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + } - assertEquals(numSSTables, cfs.getSSTables().size()); ++ assertEquals(numSSTables, cfs.getLiveSSTables().size()); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
