This is an automated email from the ASF dual-hosted git repository. paulo pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 951645a1fc4486f2b941c4eb72c429780badca07 Merge: 3b1ce69 5f50c79 Author: Paulo Motta <[email protected]> AuthorDate: Mon Mar 14 17:21:51 2022 -0300 Merge branch 'cassandra-4.0' into trunk CHANGES.txt | 1 + src/java/org/apache/cassandra/db/Directories.java | 6 +-- .../apache/cassandra/db/ColumnFamilyStoreTest.java | 41 ++++++++++++++++++ .../apache/cassandra/index/sasi/SASIIndexTest.java | 48 +++++++++++++--------- 4 files changed, 74 insertions(+), 22 deletions(-) diff --cc CHANGES.txt index c57742f,86eae2b..ea1826a --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -133,26 -46,15 +133,27 @@@ Merged from 4.0 * Avoid rewriting all sstables during cleanup when transient replication is enabled (CASSANDRA-16966) * Prevent CQLSH from failure on Python 3.10 (CASSANDRA-16987) * Avoid trying to acquire 0 permits from the rate limiter when taking snapshot (CASSANDRA-16872) - * Upgrade Caffeine to 2.5.6 (CASSANDRA-15153) - * Include SASI components to snapshots (CASSANDRA-15134) - * Fix missed wait latencies in the output of `nodetool tpstats -F` (CASSANDRA-16938) * Remove all the state pollution between tests in SSTableReaderTest (CASSANDRA-16888) * Delay auth setup until after gossip has settled to avoid unavailables on startup (CASSANDRA-16783) - * Fix clustering order logic in CREATE MATERIALIZED VIEW (CASSANDRA-16898) * org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes data twice (CASSANDRA-16900) + * Fix clustering order logic in CREATE MATERIALIZED VIEW (CASSANDRA-16898) * Exclude Jackson 1.x transitive dependency of hadoop* provided dependencies (CASSANDRA-16854) + * Tolerate missing DNS entry when completing a host replacement (CASSANDRA-16873) + * Harden PrunableArrayQueue against Pruner implementations that might throw exceptions (CASSANDRA-16866) + * Move RepairedDataInfo to the execution controller rather than the ReadCommand to avoid unintended sharing (CASSANDRA-16721) + * Bump zstd-jni version to 1.5.0-4 (CASSANDRA-16884) + * Remove assumption that all urgent messages are small (CASSANDRA-16877) + * ArrayClustering.unsharedHeapSize does not include the data so undercounts the heap size (CASSANDRA-16845) + * Improve help, doc and error messages about sstabledump -k and -x arguments (CASSANDRA-16818) + * Add repaired/unrepaired bytes back to nodetool (CASSANDRA-15282) + * Upgrade lz4-java to 1.8.0 to add RH6 support back (CASSANDRA-16753) + * Improve DiagnosticEventService.publish(event) logging message of events (CASSANDRA-16749) + * Cleanup dependency scopes (CASSANDRA-16704) + * Make JmxHistogram#getRecentValues() and JmxTimer#getRecentValues() thread-safe (CASSANDRA-16707) Merged from 3.11: ++ * Fix snapshot true size calculation (CASSANDRA-17267) + * dropping of a materialized view creates a snapshot with dropped- prefix (CASSANDRA-17415) + * Validate existence of DCs when repairing (CASSANDRA-17407) * Add key validation to ssstablescrub (CASSANDRA-16969) * Update Jackson from 2.9.10 to 2.12.5 (CASSANDRA-16851) * Make assassinate more resilient to missing tokens (CASSANDRA-16847) diff --cc src/java/org/apache/cassandra/db/Directories.java index 5a9c563,f09cdae..1cc350d --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@@ -1224,7 -1162,7 +1224,7 @@@ public class Directorie SSTableSizeSummer(File path, List<File> files) { super(path); - toSkip = new HashSet<>(files); - toSkip = files.stream().map(f -> f.getName()).collect(Collectors.toSet()); ++ toSkip = files.stream().map(f -> f.name()).collect(Collectors.toSet()); } @Override @@@ -1235,7 -1173,7 +1235,7 @@@ return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name) - && !toSkip.contains(file); - && !toSkip.contains(file.getName()); ++ && !toSkip.contains(file.name()); } } diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index e7c7e22,266b37d..d970b12 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@@ -29,10 -31,16 +29,11 @@@ import org.junit.Before import org.junit.BeforeClass; import org.junit.Test; + import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.utils.Pair; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.snapshot.SnapshotManifest; +import org.apache.cassandra.service.snapshot.TableSnapshot; import com.google.common.collect.Iterators; import org.apache.cassandra.*; @@@ -252,6 -258,46 +253,46 @@@ public class ColumnFamilyStoreTes } @Test + public void testSnapshotSize() + { + // cleanup any previous test gargbage + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + cfs.clearSnapshot(""); + + // Add row + new RowUpdateBuilder(cfs.metadata(), 0, "key1") + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + + // snapshot + cfs.snapshot("basic", null, false, false); + + // check snapshot was created - Map<String, Directories.SnapshotSizeDetails> snapshotDetails = cfs.getSnapshotDetails(); ++ Map<String, TableSnapshot> snapshotDetails = cfs.listSnapshots(); + assertThat(snapshotDetails).hasSize(1); + assertThat(snapshotDetails).containsKey("basic"); + + // check that sizeOnDisk > trueSize = 0 - Directories.SnapshotSizeDetails details = snapshotDetails.get("basic"); - assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes); - assertThat(details.dataSizeBytes).isZero(); ++ TableSnapshot details = snapshotDetails.get("basic"); ++ assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes()); ++ assertThat(details.computeTrueSizeBytes()).isZero(); + + // compact base table to make trueSize > 0 + cfs.forceMajorCompaction(); + LifecycleTransaction.waitForDeletions(); + + // sizeOnDisk > trueSize because trueSize does not include manifest.json + // Check that truesize now is > 0 - snapshotDetails = cfs.getSnapshotDetails(); ++ snapshotDetails = cfs.listSnapshots(); + details = snapshotDetails.get("basic"); - assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes); - assertThat(details.dataSizeBytes).isPositive(); ++ assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes()); ++ assertThat(details.computeTrueSizeBytes()).isPositive(); + } + + @Test public void testBackupAfterFlush() throws Throwable { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); diff --cc test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index 15dd5d8,945a5e9..10fff4b --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@@ -41,8 -42,8 +41,9 @@@ import org.apache.cassandra.cql3.CQLTes import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; + import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.index.Index; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; @@@ -164,11 -165,17 +164,16 @@@ public class SASIIndexTes try { store.snapshot(snapshotName); ++ + // Compact to make true snapshot size != 0 + store.forceMajorCompaction(); + LifecycleTransaction.waitForDeletions(); + - FileReader reader = new FileReader(store.getDirectories().getSnapshotManifestFile(snapshotName)); - JSONObject manifest = (JSONObject) new JSONParser().parse(reader); - JSONArray files = (JSONArray) manifest.get("files"); + SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName)); Assert.assertFalse(ssTableReaders.isEmpty()); - Assert.assertFalse(files.isEmpty()); - Assert.assertEquals(ssTableReaders.size(), files.size()); + Assert.assertFalse(manifest.files.isEmpty()); + Assert.assertEquals(ssTableReaders.size(), manifest.files.size()); Map<Descriptor, Set<Component>> snapshotSSTables = store.getDirectories() .sstableLister(Directories.OnTxnErr.IGNORE) @@@ -202,11 -208,11 +206,11 @@@ tableSize += componentSize; } } - - Map<String, Directories.SnapshotSizeDetails> details = store.getSnapshotDetails(); + + TableSnapshot details = store.listSnapshots().get(snapshotName); // check that SASI components are included in the computation of snapshot size - Assert.assertEquals(details.computeTrueSizeBytes(), tableSize + indexSize); - Assert.assertEquals(tableSize + indexSize, (long) details.get(snapshotName).dataSizeBytes); ++ Assert.assertEquals(tableSize + indexSize, details.computeTrueSizeBytes()); } finally { @@@ -2651,21 -2661,25 +2659,24 @@@ do { count = 0; - currentPage = getIndexed(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions); - if (currentPage == null) - break; + ReadCommand command = getIndexReadCommand(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions); - while (currentPage.hasNext()) + try (ReadExecutionController controller = command.executionController(); + UnfilteredPartitionIterator currentPage = command.executeLocally(controller)) { - try (UnfilteredRowIterator row = currentPage.next()) + if (currentPage == null) + break; + + while (currentPage.hasNext()) { - uniqueKeys.add(row.partitionKey()); - lastKey = row.partitionKey(); - count++; + try (UnfilteredRowIterator row = currentPage.next()) + { + uniqueKeys.add(row.partitionKey()); + lastKey = row.partitionKey(); + count++; + } } } -- - currentPage.close(); } while (count == pageSize); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
