Repository: cassandra Updated Branches: refs/heads/trunk 2f984e330 -> ee9e38b7d
Add --skip-flush option to nodetool snapshot patch by anubhavkale; reviewed by pauloricardomg for CASSANDRA-10907 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ee9e38b7 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ee9e38b7 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ee9e38b7 Branch: refs/heads/trunk Commit: ee9e38b7dc20a3b98e5a68904be35d727ed9c09c Parents: 2f984e3 Author: anubhavkale <anubh...@microsoft.com> Authored: Tue Jan 19 12:56:29 2016 -0800 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Fri Jan 22 16:42:55 2016 +0100 ---------------------------------------------------------------------- CHANGES.txt | 2 +- .../apache/cassandra/db/ColumnFamilyStore.java | 21 +++- src/java/org/apache/cassandra/db/Keyspace.java | 18 +++- .../repair/RepairMessageVerbHandler.java | 2 +- .../cassandra/service/StorageService.java | 106 +++++++++++++------ .../cassandra/service/StorageServiceMBean.java | 27 ++--- .../org/apache/cassandra/tools/NodeProbe.java | 15 +-- .../cassandra/tools/nodetool/Snapshot.java | 15 ++- .../cassandra/db/ColumnFamilyStoreTest.java | 4 +- .../service/StorageServiceServerTest.java | 13 ++- 10 files changed, 156 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3f80264..b8314e5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.4 + * Add --skip-flush option to nodetool snapshot * Skip values for non-queried columns (CASSANDRA-10657) * Add support for secondary indexes on static columns (CASSANDRA-8103) * CommitLogUpgradeTestMaker creates broken commit logs (CASSANDRA-11051) @@ -9,7 +10,6 @@ * Add nodetool gettimeout and settimeout commands (CASSANDRA-10953) * Add 3.0 metadata to sstablemetadata output (CASSANDRA-10838) - 3.3 * Avoid bootstrap hanging when existing nodes have no data to stream (CASSANDRA-11010) Merged from 3.0: http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 486808b..52d8747 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1749,16 +1749,31 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean */ public Set<SSTableReader> snapshot(String snapshotName) { - return snapshot(snapshotName, null, false); + return snapshot(snapshotName, false); + } + + /** + * Take a snap shot of this columnfamily store. + * + * @param snapshotName the name of the associated with the snapshot + * @param skipFlush Skip blocking flush of memtable + */ + public Set<SSTableReader> snapshot(String snapshotName, boolean skipFlush) + { + return snapshot(snapshotName, null, false, skipFlush); } /** * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup + * @param skipFlush Skip blocking flush of memtable */ - public Set<SSTableReader> snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral) + public Set<SSTableReader> snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipFlush) { - forceBlockingFlush(); + if (!skipFlush) + { + forceBlockingFlush(); + } return snapshotWithoutFlush(snapshotName, predicate, ephemeral); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/db/Keyspace.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index 0ec94ea..6122479 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -219,9 +219,10 @@ public class Keyspace * * @param snapshotName the tag associated with the name of the snapshot. This value may not be null * @param columnFamilyName the column family to snapshot or all on null + * @param skipFlush Skip blocking flush of memtable * @throws IOException if the column family doesn't exist */ - public void snapshot(String snapshotName, String columnFamilyName) throws IOException + public void snapshot(String snapshotName, String columnFamilyName, boolean skipFlush) throws IOException { assert snapshotName != null; boolean tookSnapShot = false; @@ -230,7 +231,7 @@ public class Keyspace if (columnFamilyName == null || cfStore.name.equals(columnFamilyName)) { tookSnapShot = true; - cfStore.snapshot(snapshotName); + cfStore.snapshot(snapshotName, skipFlush); } } @@ -239,6 +240,19 @@ public class Keyspace } /** + * Take a snapshot of the specific column family, or the entire set of column families + * if columnFamily is null with a given timestamp + * + * @param snapshotName the tag associated with the name of the snapshot. This value may not be null + * @param columnFamilyName the column family to snapshot or all on null + * @throws IOException if the column family doesn't exist + */ + public void snapshot(String snapshotName, String columnFamilyName) throws IOException + { + snapshot(snapshotName, columnFamilyName, false); + } + + /** * @param clientSuppliedName may be null. * @return the name of the snapshot */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 9ac859f..6c04729 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -90,7 +90,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage> !sstable.metadata.isIndex() && // exclude SSTables from 2i new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(repairingRange); } - }, true); //ephemeral snapshot, if repair fails, it will be cleaned next startup + }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfs.metadata.cfId, desc.parentSessionId); if (!Sets.intersection(currentlyRepairing, snapshottedSSSTables).isEmpty()) http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index fd6436d..932d949 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -2775,6 +2775,47 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } /** + * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * + * @param tag + * the tag given to the snapshot; may not be null or empty + * @param options + * Map of options (skipFlush is the only supported option for now) + * @param entities + * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + */ + @Override + public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException + { + boolean skipFlush = Boolean.parseBoolean(options.getOrDefault("skipFlush", "false")); + + if (entities != null && entities.length > 0 && entities[0].contains(".")) + { + takeMultipleTableSnapshot(tag, skipFlush, entities); + } + else + { + takeSnapshot(tag, skipFlush, entities); + } + } + + /** + * Takes the snapshot of a specific table. A snapshot name must be + * specified. + * + * @param keyspaceName + * the keyspace which holds the specified table + * @param tableName + * the table to snapshot + * @param tag + * the tag given to the snapshot; may not be null or empty + */ + public void takeTableSnapshot(String keyspaceName, String tableName, String tag) + throws IOException { + takeMultipleTableSnapshot(tag, false, keyspaceName + "." + tableName); + } + + /** * Takes the snapshot for the given keyspaces. A snapshot name must be specified. * * @param tag the tag given to the snapshot; may not be null or empty @@ -2782,6 +2823,32 @@ public class StorageService extends NotificationBroadcasterSupport implements IE */ public void takeSnapshot(String tag, String... keyspaceNames) throws IOException { + takeSnapshot(tag, false, keyspaceNames); + } + + /** + * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * + * @param tag + * the tag given to the snapshot; may not be null or empty + * @param tableList + * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 + */ + public void takeMultipleTableSnapshot(String tag, String... tableList) + throws IOException + { + takeMultipleTableSnapshot(tag, false, tableList); + } + + /** + * Takes the snapshot for the given keyspaces. A snapshot name must be specified. + * + * @param tag the tag given to the snapshot; may not be null or empty + * @param skipFlush Skip blocking flush of memtable + * @param keyspaceNames the names of the keyspaces to snapshot; empty means "all." + */ + private void takeSnapshot(String tag, boolean skipFlush, String... keyspaceNames) throws IOException + { if (operationMode == Mode.JOINING) throw new IOException("Cannot snapshot until bootstrap completes"); if (tag == null || tag.equals("")) @@ -2807,37 +2874,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (Keyspace keyspace : keyspaces) - keyspace.snapshot(tag, null); - } - - /** - * Takes the snapshot of a specific table. A snapshot name must be specified. - * - * @param keyspaceName the keyspace which holds the specified table - * @param tableName the table to snapshot - * @param tag the tag given to the snapshot; may not be null or empty - */ - public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException - { - if (keyspaceName == null) - throw new IOException("You must supply a keyspace name"); - if (operationMode == Mode.JOINING) - throw new IOException("Cannot snapshot until bootstrap completes"); - - if (tableName == null) - throw new IOException("You must supply a table name"); - if (tableName.contains(".")) - throw new IllegalArgumentException("Cannot take a snapshot of a secondary index by itself. Run snapshot on the table that owns the index."); - - if (tag == null || tag.equals("")) - throw new IOException("You must supply a snapshot name."); - - Keyspace keyspace = getValidKeyspace(keyspaceName); - ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); - if (columnFamilyStore.snapshotExists(tag)) - throw new IOException("Snapshot " + tag + " already exists."); - - columnFamilyStore.snapshot(tag); + keyspace.snapshot(tag, null, skipFlush); } /** @@ -2846,11 +2883,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * * @param tag * the tag given to the snapshot; may not be null or empty + * @param skipFlush + * Skip blocking flush of memtable * @param tableList * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 */ - @Override - public void takeMultipleTableSnapshot(String tag, String... tableList) + private void takeMultipleTableSnapshot(String tag, boolean skipFlush, String... tableList) throws IOException { Map<Keyspace, List<String>> keyspaceColumnfamily = new HashMap<Keyspace, List<String>>(); @@ -2899,7 +2937,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (Entry<Keyspace, List<String>> entry : keyspaceColumnfamily.entrySet()) { for (String table : entry.getValue()) - entry.getKey().snapshot(tag, table); + entry.getKey().snapshot(tag, table, skipFlush); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/service/StorageServiceMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 77bcdb7..dc12253 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -194,31 +194,34 @@ public interface StorageServiceMBean extends NotificationEmitter public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key); /** - * Takes the snapshot for the given keyspaces. A snapshot name must be specified. - * - * @param tag the tag given to the snapshot; may not be null or empty - * @param keyspaceNames the name of the keyspaces to snapshot; empty means "all." + * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead. */ + @Deprecated public void takeSnapshot(String tag, String... keyspaceNames) throws IOException; /** - * Takes the snapshot of a specific column family. A snapshot name must be specified. - * - * @param keyspaceName the keyspace which holds the specified column family - * @param tableName the table to snapshot - * @param tag the tag given to the snapshot; may not be null or empty + * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead. */ + @Deprecated public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException; /** + * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead. + */ + @Deprecated + public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException; + + /** * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. * * @param tag * the tag given to the snapshot; may not be null or empty - * @param tableList - * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 + * @param options + * Map of options (skipFlush is the only supported option for now) + * @param entities + * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... */ - public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException; + public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException; /** * Remove the snapshot with the given name from the given keyspaces. http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 2bc516a..405f70c 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -519,9 +519,10 @@ public class NodeProbe implements AutoCloseable * * @param snapshotName the name of the snapshot. * @param table the table to snapshot or all on null + * @param options Options (skipFlush for now) * @param keyspaces the keyspaces to snapshot */ - public void takeSnapshot(String snapshotName, String table, String... keyspaces) throws IOException + public void takeSnapshot(String snapshotName, String table, Map<String, String> options, String... keyspaces) throws IOException { if (table != null) { @@ -529,10 +530,11 @@ public class NodeProbe implements AutoCloseable { throw new IOException("When specifying the table for a snapshot, you must specify one and only one keyspace"); } - ssProxy.takeTableSnapshot(keyspaces[0], table, snapshotName); + + ssProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table); } else - ssProxy.takeSnapshot(snapshotName, keyspaces); + ssProxy.takeSnapshot(snapshotName, options, keyspaces); } /** @@ -540,21 +542,22 @@ public class NodeProbe implements AutoCloseable * * @param snapshotName * the name of the snapshot. + * @param options + * Options (skipFlush for now) * @param tableList * list of columnfamily from different keyspace in the form of ks1.cf1 ks2.cf2 */ - public void takeMultipleTableSnapshot(String snapshotName, String... tableList) + public void takeMultipleTableSnapshot(String snapshotName, Map<String, String> options, String... tableList) throws IOException { if (null != tableList && tableList.length != 0) { - ssProxy.takeMultipleTableSnapshot(snapshotName, tableList); + ssProxy.takeSnapshot(snapshotName, options, tableList); } else { throw new IOException("The column family List for a snapshot should not be empty or null"); } - } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java index 4f549e5..8941ec1 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java @@ -25,7 +25,9 @@ import io.airlift.command.Option; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; @@ -45,6 +47,9 @@ public class Snapshot extends NodeToolCmd @Option(title = "ktlist", name = { "-kt", "--kt-list", "-kc", "--kc.list" }, description = "The list of Keyspace.table to take snapshot.(you must not specify only keyspace)") private String ktList = null; + @Option(title = "skip-flush", name = {"-sf", "--skip-flush"}, description = "Do not flush memtables before snapshotting (snapshot will not contain unflushed data)") + private boolean skipFlush = false; + @Override public void execute(NodeProbe probe) { @@ -54,6 +59,9 @@ public class Snapshot extends NodeToolCmd sb.append("Requested creating snapshot(s) for "); + Map<String, String> options = new HashMap<String,String>(); + options.put("skipFlush", Boolean.toString(skipFlush)); + // Create a separate path for kclist to avoid breaking of already existing scripts if (null != ktList && !ktList.isEmpty()) { @@ -67,8 +75,9 @@ public class Snapshot extends NodeToolCmd } if (!snapshotName.isEmpty()) sb.append(" with snapshot name [").append(snapshotName).append("]"); + sb.append(" and options ").append(options.toString()); System.out.println(sb.toString()); - probe.takeMultipleTableSnapshot(snapshotName, ktList.split(",")); + probe.takeMultipleTableSnapshot(snapshotName, options, ktList.split(",")); System.out.println("Snapshot directory: " + snapshotName); } else @@ -80,10 +89,10 @@ public class Snapshot extends NodeToolCmd if (!snapshotName.isEmpty()) sb.append(" with snapshot name [").append(snapshotName).append("]"); - + sb.append(" and options ").append(options.toString()); System.out.println(sb.toString()); - probe.takeSnapshot(snapshotName, table, toArray(keyspaces, String.class)); + probe.takeSnapshot(snapshotName, table, options, toArray(keyspaces, String.class)); System.out.println("Snapshot directory: " + snapshotName); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 6840e2b..af43152 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -321,8 +321,8 @@ public class ColumnFamilyStoreTest } ScrubTest.fillIndexCF(cfs, false, colValues); - cfs.snapshot("nonEphemeralSnapshot", null, false); - cfs.snapshot("ephemeralSnapshot", null, true); + cfs.snapshot("nonEphemeralSnapshot", null, false, false); + cfs.snapshot("ephemeralSnapshot", null, true, false); Map<String, Pair<Long, Long>> snapshotDetails = cfs.getSnapshotDetails(); assertEquals(2, snapshotDetails.size()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index 392b6f4..e438a6b 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -94,10 +94,10 @@ public class StorageServiceServerTest } @Test - public void testSnapshot() throws IOException + public void testSnapshotWithFlush() throws IOException { // no need to insert extra data, even an "empty" database will have a little information in the system keyspace - StorageService.instance.takeSnapshot("snapshot"); + StorageService.instance.takeSnapshot(UUID.randomUUID().toString()); } private void checkTempFilePresence(File f, boolean exist) @@ -173,7 +173,14 @@ public class StorageServiceServerTest public void testTableSnapshot() throws IOException { // no need to insert extra data, even an "empty" database will have a little information in the system keyspace - StorageService.instance.takeTableSnapshot(SchemaKeyspace.NAME, SchemaKeyspace.KEYSPACES, "cf_snapshot"); + StorageService.instance.takeTableSnapshot(SchemaKeyspace.NAME, SchemaKeyspace.KEYSPACES, UUID.randomUUID().toString()); + } + + @Test + public void testSnapshot() throws IOException + { + // no need to insert extra data, even an "empty" database will have a little information in the system keyspace + StorageService.instance.takeSnapshot(UUID.randomUUID().toString(), SchemaKeyspace.NAME); } @Test