This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 4f11cb9feebdb0017de1e1063cbf72b840567518
Merge: 979a71b 257fb03
Author: adelapena <a.penya.gar...@gmail.com>
AuthorDate: Fri Jul 10 18:04:18 2020 +0100
Merge branch 'cassandra-2.2' into cassandra-3.0
# Conflicts:
# CHANGES.txt
# src/java/org/apache/cassandra/db/ColumnFamilyStore.java
# test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
CHANGES.txt | 3 ++
.../org/apache/cassandra/db/ColumnFamilyStore.java | 12 +++----
src/java/org/apache/cassandra/db/Directories.java | 23 ++++++++-----
.../apache/cassandra/io/sstable/Descriptor.java | 7 +++-
.../apache/cassandra/db/ColumnFamilyStoreTest.java | 40 ++++++++++++++++++++++
5 files changed, 69 insertions(+), 16 deletions(-)
diff --cc CHANGES.txt
index 16974c4,02de7c1..5681007
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -26,71 -4,24 +26,74 @@@ Merged from 2.2
* Remove duplicated tools binaries from tarballs (CASSANDRA-15768)
* Duplicate results with DISTINCT queries in mixed mode (CASSANDRA-15501)
* Disable JMX rebinding (CASSANDRA-15653)
- * Fix Commit log replays when static column clustering keys are collections
(CASSANDRA-14365)
- * Fix Red Hat init script on newer systemd versions (CASSANDRA-15273)
- * Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567)
Merged from 2.1:
+ * Fix writing of snapshot manifest when the table has table-backed secondary
indexes (CASSANDRA-10968)
* Fix parse error in cqlsh COPY FROM and formatting for map of blobs
(CASSANDRA-15679)
+ * Fix Commit log replays when static column clustering keys are collections
(CASSANDRA-14365)
+ * Fix Red Hat init script on newer systemd versions (CASSANDRA-15273)
+ * Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567)
+
-2.2.16
+3.0.20
+ * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506)
+ * Include updates to static column in mutation size calculations
(CASSANDRA-15293)
+ * Fix point-in-time recoevery ignoring timestamp of updates to static
columns (CASSANDRA-15292)
+ * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306)
+ * Fix sstabledump's position key value when partitions have multiple rows
(CASSANDRA-14721)
+ * Avoid over-scanning data directories in LogFile.verify() (CASSANDRA-15364)
+ * Bump generations and document changes to system_distributed and
system_traces in 3.0, 3.11
+ (CASSANDRA-15441)
+ * Fix system_traces creation timestamp; optimise system keyspace upgrades
(CASSANDRA-15398)
+ * Fix various data directory prefix matching issues (CASSANDRA-13974)
+ * Minimize clustering values in metadata collector (CASSANDRA-15400)
+ * Avoid over-trimming of results in mixed mode clusters (CASSANDRA-15405)
+ * validate value sizes in LegacyLayout (CASSANDRA-15373)
+ * Ensure that tracing doesn't break connections in 3.x/4.0 mixed mode by
default (CASSANDRA-15385)
+ * Make sure index summary redistribution does not start when compactions are
paused (CASSANDRA-15265)
+ * Ensure legacy rows have primary key livenessinfo when they contain illegal
cells (CASSANDRA-15365)
+ * Fix race condition when setting bootstrap flags (CASSANDRA-14878)
+ * Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426)
+Merged from 2.2:
* Fix SELECT JSON output for empty blobs (CASSANDRA-15435)
* In-JVM DTest: Set correct internode message version for upgrade test
(CASSANDRA-15371)
- * In-JVM DTest: Support NodeTool in dtest
+ * In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429)
+
-2.2.15
+3.0.19
+ * Add ability to cap max negotiable protocol version (CASSANDRA-15193)
+ * Gossip tokens on startup if available (CASSANDRA-15335)
+ * Fix resource leak in CompressedSequentialWriter (CASSANDRA-15340)
+ * Fix merge which reverted CASSANDRA-14993 (CASSANDRA-15289)
+ * Fix LegacyLayout RangeTombstoneList IndexOutOfBoundsException when
upgrading and RangeTombstone bounds are asymmetric (CASSANDRA-15172)
+ * Fix NPE when using allocate_tokens_for_keyspace on new DC/rack
(CASSANDRA-14952)
+ * Filter sstables earlier when running cleanup (CASSANDRA-15100)
+ * Use mean row count instead of mean column count for index selectivity
calculation (CASSANDRA-15259)
+ * Avoid updating unchanged gossip states (CASSANDRA-15097)
+ * Prevent recreation of previously dropped columns with a different kind
(CASSANDRA-14948)
+ * Prevent client requests from blocking on executor task queue
(CASSANDRA-15013)
+ * Toughen up column drop/recreate type validations (CASSANDRA-15204)
+ * LegacyLayout should handle paging states that cross a collection column
(CASSANDRA-15201)
+ * Prevent RuntimeException when username or password is empty/null
(CASSANDRA-15198)
+ * Multiget thrift query returns null records after digest mismatch
(CASSANDRA-14812)
+ * Skipping illegal legacy cells can break reverse iteration of indexed
partitions (CASSANDRA-15178)
+ * Handle paging states serialized with a different version than the
session's (CASSANDRA-15176)
+ * Throw IOE instead of asserting on unsupporter peer versions
(CASSANDRA-15066)
+ * Update token metadata when handling MOVING/REMOVING_TOKEN events
(CASSANDRA-15120)
+ * Add ability to customize cassandra log directory using $CASSANDRA_LOG_DIR
(CASSANDRA-15090)
+ * Skip cells with illegal column names when reading legacy sstables
(CASSANDRA-15086)
+ * Fix assorted gossip races and add related runtime checks (CASSANDRA-15059)
+ * Fix mixed mode partition range scans with limit (CASSANDRA-15072)
+ * cassandra-stress works with frozen collections: list and set
(CASSANDRA-14907)
+ * For nodetool listsnapshots output, put spaces between columns, and
increase snapshot padding (CASSANDRA-14876)
+ * Fix handling FS errors on writing and reading flat files - LogTransaction
and hints (CASSANDRA-15053)
+ * Avoid double closing the iterator to avoid overcounting the number of
requests (CASSANDRA-15058)
+ * Improve `nodetool status -r` speed (CASSANDRA-14847)
+ * Improve merkle tree size and time on heap (CASSANDRA-14096)
+ * Add missing commands to nodetool-completion (CASSANDRA-14916)
+ * Anti-compaction temporarily corrupts sstable state for readers
(CASSANDRA-15004)
+ Merged from 2.2:
* Catch non-IOException in FileUtils.close to make sure that all resources
are closed (CASSANDRA-15225)
* Handle exceptions during authentication/authorization (CASSANDRA-15041)
- * Fix JDK7 compatibility broken in cassandra-2.2 (CASSANDRA-15050)
* Support cross version messaging in in-jvm upgrade dtests (CASSANDRA-15078)
* Fix index summary redistribution cancellation (CASSANDRA-15045)
* Refactor Circle CI configuration (CASSANDRA-14806)
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index a9bd54a,7f625e1..70c14c0
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1640,10 -2426,10 +1640,10 @@@ public class ColumnFamilyStore implemen
public Set<SSTableReader> snapshotWithoutFlush(String snapshotName,
Predicate<SSTableReader> predicate, boolean ephemeral)
{
Set<SSTableReader> snapshottedSSTables = new HashSet<>();
+ final JSONArray filesJSONArr = new JSONArray();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
- final JSONArray filesJSONArr = new JSONArray();
- try (RefViewFragment currentView =
cfs.selectAndReference(CANONICAL_SSTABLES))
+ try (RefViewFragment currentView =
cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate ==
null || predicate.apply(x))))
{
for (SSTableReader ssTable : currentView.sstables)
{
@@@ -1655,13 -2444,9 +1655,13 @@@
logger.trace("Snapshot for {} keyspace data file {}
created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
snapshottedSSTables.add(ssTable);
}
-
- writeSnapshotManifest(filesJSONArr, snapshotName);
-
- if (!Schema.isLocalSystemKeyspace(metadata.ksName) &&
!Schema.isReplicatedSystemKeyspace(metadata.ksName))
- writeSnapshotSchema(snapshotName);
}
}
+ writeSnapshotManifest(filesJSONArr, snapshotName);
++
++ if (!Schema.isLocalSystemKeyspace(metadata.ksName) &&
!Schema.isReplicatedSystemKeyspace(metadata.ksName))
++ writeSnapshotSchema(snapshotName);
++
if (ephemeral)
createEphemeralSnapshotMarkerFile(snapshotName);
return snapshottedSSTables;
diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 6840e2b,4d452c6..f7152ff
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@@ -19,38 -19,98 +19,44 @@@
package org.apache.cassandra.db;
import java.io.File;
+ import java.io.FileReader;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
-import org.apache.cassandra.db.index.PerRowSecondaryIndexTest;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
+import java.util.*;
+
+import org.junit.Before;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
++import org.json.simple.JSONArray;
++import org.json.simple.JSONObject;
++import org.json.simple.parser.JSONParser;
++
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Iterators;
+import org.apache.cassandra.*;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
-import org.apache.cassandra.db.composites.CellName;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.composites.CellNames;
-import org.apache.cassandra.db.composites.Composites;
-import org.apache.cassandra.db.filter.ColumnSlice;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.NamesQueryFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.SliceQueryFilter;
-import org.apache.cassandra.db.index.SecondaryIndex;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.LexicalUUIDType;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.ExcludingBounds;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.IncludingExcludingBounds;
-import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.ClearableHistogram;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
-import org.apache.cassandra.thrift.ThriftValidation;
+import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.WrappedRunnable;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-
-import static org.apache.cassandra.Util.cellname;
-import static org.apache.cassandra.Util.column;
-import static org.apache.cassandra.Util.dk;
-import static org.apache.cassandra.Util.rp;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
+import static junit.framework.Assert.assertNotNull;
+
@RunWith(OrderedJUnit4ClassRunner.class)
public class ColumnFamilyStoreTest
{
@@@ -512,30 -2362,37 +518,64 @@@
}
@Test
+ public void testSnapshotWithoutFlushWithSecondaryIndexes() throws
Exception
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1);
+ cfs.truncateBlocking();
+
- List<Mutation> rms = new LinkedList<>();
- Mutation rm;
-
- rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1"));
- rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 0);
- rm.add(CF_INDEX1, cellname("nobirthdate"), ByteBufferUtil.bytes(1L),
0);
- rms.add(rm);
- Util.writeColumnFamily(rms);
++ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, "key")
++ .newRow()
++ .add("birthdate", 1L)
++ .add("notbirthdate", 2L);
++ new Mutation(builder.build()).applyUnsafe();
++ cfs.forceBlockingFlush();
+
+ String snapshotName = "newSnapshot";
+ cfs.snapshotWithoutFlush(snapshotName);
+
- File snapshotManifestFile =
cfs.directories.getSnapshotManifestFile(snapshotName);
++ File snapshotManifestFile =
cfs.getDirectories().getSnapshotManifestFile(snapshotName);
+ JSONParser parser = new JSONParser();
+ JSONObject manifest = (JSONObject) parser.parse(new
FileReader(snapshotManifestFile));
+ JSONArray files = (JSONArray) manifest.get("files");
+
+ // Keyspace1-Indexed1 and the corresponding index
+ assert files.size() == 2;
+
+ // Snapshot of the secondary index is stored in the subfolder with
the same file name
+ String baseTableFile = (String) files.get(0);
+ String indexTableFile = (String) files.get(1);
+ assert !baseTableFile.equals(indexTableFile);
+ assert Directories.isSecondaryIndexFolder(new
File(indexTableFile).getParentFile());
+ assert indexTableFile.endsWith(baseTableFile);
+ }
++
++
++ @Test
+ public void testScrubDataDirectories() throws Throwable
+ {
+ ColumnFamilyStore cfs =
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+
+ ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
+
+ new RowUpdateBuilder(cfs.metadata, 2,
"key").clustering("name").add("val", "2").build().applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ // Nuke the metadata and reload that sstable
+ Collection<SSTableReader> ssTables = cfs.getLiveSSTables();
+ assertEquals(1, ssTables.size());
+ SSTableReader ssTable = ssTables.iterator().next();
+
+ String dataFileName = ssTable.descriptor.filenameFor(Component.DATA);
+ String tmpDataFileName =
ssTable.descriptor.tmpFilenameFor(Component.DATA);
+ new File(dataFileName).renameTo(new File(tmpDataFileName));
+
+ ssTable.selfRef().release();
+
+ ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
+
+ List<File> ssTableFiles = new
Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
+ assertNotNull(ssTableFiles);
+ assertEquals(0, ssTableFiles.size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
