This is an automated email from the ASF dual-hosted git repository.
smiklosovic pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new c26dc06a28 add datapaths subcommand to nodetool
c26dc06a28 is described below
commit c26dc06a28b0e150384474001ac23026ae76e6d5
Author: Tibor Répási <[email protected]>
AuthorDate: Wed Apr 20 22:10:13 2022 +0200
add datapaths subcommand to nodetool
patch by Tibor Repasi; reviewed by Stefan Miklosovic and Brandon Williams
for CASSANDRA-17568
---
CHANGES.txt | 1 +
.../pages/troubleshooting/use_nodetool.adoc | 43 ++++++
src/java/org/apache/cassandra/tools/NodeTool.java | 1 +
.../apache/cassandra/tools/nodetool/DataPaths.java | 53 +++++++
.../tools/nodetool/stats/DataPathsHolder.java | 84 ++++++++++
.../tools/nodetool/stats/DataPathsPrinter.java | 63 ++++++++
.../cassandra/tools/nodetool/DataPathsTest.java | 170 +++++++++++++++++++++
7 files changed, 415 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index 5ab33a229b..a1213090e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.1
+ * Tool to list data paths of existing tables (CASSANDRA-17568)
* Migrate track_warnings to more standard naming conventions and use latest
configuration types rather than long (CASSANDRA-17560)
* Add support for CONTAINS and CONTAINS KEY in conditional UPDATE and DELETE
statement (CASSANDRA-10537)
* Migrate advanced config parameters to the new Config types (CASSANDRA-17431)
diff --git a/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc
b/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc
index f80d039695..a313432cbb 100644
--- a/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc
+++ b/doc/modules/cassandra/pages/troubleshooting/use_nodetool.adoc
@@ -240,3 +240,46 @@ concurrent compactions such that compactions complete
quickly but don't
take too many resources away from query threads is very important for
performance. If you notice compaction unable to keep up, try tuning
Cassandra's `concurrent_compactors` or `compaction_throughput` options.
+
+[[nodetool-datapaths]]
+== Paths used for data files
+
+Cassandra is persisting data on disk within the configured directories. Data
+files are distributed among the directories configured with
`data_file_directories`.
+Resembling the structure of keyspaces and tables, Cassandra is creating
+subdirectories within `data_file_directories`. However, directories aren't
removed
+even if the tables and keyspaces are dropped. While these directories are kept
with
+the reason of holding snapshots, they are subject to removal. This is where
operators
+need to know which directories are still in use. Running the `nodetool
datapaths`
+command is an easy way to list in which directories Cassandra is actually
storing
+sstable data on disk.
+
+[source, bash]
+----
+% nodetool datapaths -- system_auth
+Keyspace: system_auth
+ Table: role_permissions
+ Paths:
+
/var/lib/cassandra/data/system_auth/role_permissions-3afbe79f219431a7add7f5ab90d8ec9c
+
+ Table: network_permissions
+ Paths:
+
/var/lib/cassandra/data/system_auth/network_permissions-d46780c22f1c3db9b4c1b8d9fbc0cc23
+
+ Table: resource_role_permissons_index
+ Paths:
+
/var/lib/cassandra/data/system_auth/resource_role_permissons_index-5f2fbdad91f13946bd25d5da3a5c35ec
+
+ Table: roles
+ Paths:
+
/var/lib/cassandra/data/system_auth/roles-5bc52802de2535edaeab188eecebb090
+
+ Table: role_members
+ Paths:
+
/var/lib/cassandra/data/system_auth/role_members-0ecdaa87f8fb3e6088d174fb36fe5c0d
+
+----
+
+By default all keyspaces and tables are listed, however, a list of `keyspace`
and
+`keyspace.table` arguments can be given to query specific data paths. Using
the `--format`
+option the output can be formatted as YAML or JSON.
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java
b/src/java/org/apache/cassandra/tools/NodeTool.java
index f9422bdbba..476353fee0 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -102,6 +102,7 @@ public class NodeTool
Compact.class,
CompactionHistory.class,
CompactionStats.class,
+ DataPaths.class,
Decommission.class,
DescribeCluster.class,
DescribeRing.class,
diff --git a/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java
b/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java
new file mode 100644
index 0000000000..10ae01e8da
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/DataPaths.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import org.apache.cassandra.tools.nodetool.stats.DataPathsHolder;
+import org.apache.cassandra.tools.nodetool.stats.DataPathsPrinter;
+import org.apache.cassandra.tools.nodetool.stats.StatsPrinter;
+
+@Command(name = "datapaths", description = "Print all directories where data
of tables are stored")
+public class DataPaths extends NodeToolCmd
+{
+ @Arguments(usage = "[<keyspace.table>...]", description = "List of table
(or keyspace) names")
+ private List<String> tableNames = new ArrayList<>();
+
+ @Option(title = "format", name = {"-F", "--format"}, description = "Output
format (json, yaml)")
+ private String outputFormat = "";
+
+ @Override
+ protected void execute(NodeProbe probe)
+ {
+ if (!outputFormat.isEmpty() && !"json".equals(outputFormat) &&
!"yaml".equals(outputFormat))
+ {
+ throw new IllegalArgumentException("arguments for -F are yaml and
json only.");
+ }
+
+ DataPathsHolder holder = new DataPathsHolder(probe, tableNames);
+ StatsPrinter<DataPathsHolder> printer =
DataPathsPrinter.from(outputFormat);
+ printer.print(holder, probe.output().out);
+ }
+}
diff --git
a/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java
b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java
new file mode 100644
index 0000000000..9bf7bdb45f
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsHolder.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Throwables.getStackTraceAsString;
+
+import org.apache.cassandra.db.ColumnFamilyStoreMBean;
+import org.apache.cassandra.tools.NodeProbe;
+
+public class DataPathsHolder implements StatsHolder
+{
+ public final Map<String, Object> pathsHash = new HashMap<>();
+
+ public DataPathsHolder(NodeProbe probe, List<String> tableNames)
+ {
+ Iterator<Map.Entry<String, ColumnFamilyStoreMBean>> mbeansIterator =
probe.getColumnFamilyStoreMBeanProxies();
+ while (mbeansIterator.hasNext())
+ {
+ Map.Entry<String, ColumnFamilyStoreMBean> entry =
mbeansIterator.next();
+ String keyspaceName = entry.getKey();
+ String tableName = entry.getValue().getTableName();
+
+ if (!(tableNames.isEmpty() ||
+ tableNames.contains(keyspaceName + '.' + tableName) ||
+ tableNames.contains(keyspaceName) ))
+ {
+ continue;
+ }
+
+ Map<String, List<String>> ksPaths;
+ List<String> dataPaths;
+
+ try
+ {
+ dataPaths = entry.getValue().getDataPaths();
+ }
+ catch (Throwable e)
+ {
+ probe.output().err.println("Failed to get data paths for " +
keyspaceName + '.' + tableName + ". Skipped.");
+ probe.output().err.println("error: " + e.getMessage());
+ probe.output().err.println("-- StackTrace --");
+ probe.output().err.println(getStackTraceAsString(e));
+ continue;
+ }
+
+ if (pathsHash.containsKey(keyspaceName))
+ {
+ ksPaths = (Map<String, List<String>>)
pathsHash.get(keyspaceName);
+ }
+ else
+ {
+ ksPaths = new HashMap<>();
+ pathsHash.put(keyspaceName, ksPaths);
+ }
+ ksPaths.put(tableName, dataPaths);
+ }
+ }
+
+ @Override
+ public Map<String, Object> convert2Map()
+ {
+ return pathsHash;
+ }
+}
diff --git
a/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java
b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java
new file mode 100644
index 0000000000..ba2b29e766
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/DataPathsPrinter.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool.stats;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class DataPathsPrinter<T extends StatsHolder>
+{
+ public static StatsPrinter<DataPathsHolder> from(String format)
+ {
+ if ("json".equals(format))
+ return new StatsPrinter.JsonPrinter<>();
+ if ("yaml".equals(format))
+ return new StatsPrinter.YamlPrinter<>();
+
+ return new DefaultPrinter();
+ }
+
+ public static class DefaultPrinter implements StatsPrinter<DataPathsHolder>
+ {
+ @Override
+ public void print(DataPathsHolder data, PrintStream out)
+ {
+ Iterator<Map.Entry<String, Object>> iterator =
data.pathsHash.entrySet().iterator();
+
+ while (iterator.hasNext())
+ {
+ Map.Entry<String, Object> entry = iterator.next();
+
+ out.println("Keyspace: " + entry.getKey());
+ Map<String, List<String>> ksPaths = (Map<String,
List<String>>) entry.getValue();
+ for (Map.Entry<String, List<String>> table :
ksPaths.entrySet())
+ {
+ out.println("\tTable: " + table.getKey());
+ out.println("\tPaths:");
+
+ for (String path : table.getValue())
+ out.println("\t\t" + path);
+
+ out.println("");
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java
b/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java
new file mode 100644
index 0000000000..d09aceb3af
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/nodetool/DataPathsTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.tools.ToolRunner;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DataPathsTest extends CQLTester
+{
+ private static final String SUBCOMMAND = "datapaths";
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ requireNetwork();
+ startJMXServer();
+ }
+
+ @Test
+ public void testHelp()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("help",
SUBCOMMAND);
+ tool.assertOnCleanExit();
+
+ String help = "NAME\n" +
+ " nodetool datapaths - Print all directories
where data of tables are\n" +
+ " stored\n" +
+ '\n' +
+ "SYNOPSIS\n" +
+ " nodetool [(-h <host> | --host <host>)] [(-p
<port> | --port <port>)]\n" +
+ " [(-pp | --print-port)] [(-pw <password>
| --password <password>)]\n" +
+ " [(-pwf <passwordFilePath> |
--password-file <passwordFilePath>)]\n" +
+ " [(-u <username> | --username
<username>)] datapaths\n" +
+ " [(-F <format> | --format <format>)]
[--] [<keyspace.table>...]\n" +
+ '\n' +
+ "OPTIONS\n" +
+ " -F <format>, --format <format>\n" +
+ " Output format (json, yaml)\n" +
+ '\n' +
+ " -h <host>, --host <host>\n" +
+ " Node hostname or ip address\n" +
+ '\n' +
+ " -p <port>, --port <port>\n" +
+ " Remote jmx agent port number\n" +
+ '\n' +
+ " -pp, --print-port\n" +
+ " Operate in 4.0 mode with hosts
disambiguated by port number\n" +
+ '\n' +
+ " -pw <password>, --password <password>\n" +
+ " Remote jmx agent password\n" +
+ '\n' +
+ " -pwf <passwordFilePath>, --password-file
<passwordFilePath>\n" +
+ " Path to the JMX password file\n" +
+ '\n' +
+ " -u <username>, --username <username>\n" +
+ " Remote jmx agent username\n" +
+ '\n' +
+ " --\n" +
+ " This option can be used to separate
command-line options from the\n" +
+ " list of argument, (useful when arguments
might be mistaken for\n" +
+ " command-line options\n" +
+ '\n' +
+ " [<keyspace.table>...]\n" +
+ " List of table (or keyspace) names\n" +
+ '\n' +
+ '\n';
+ assertThat(tool.getStdout()).isEqualTo(help);
+ }
+
+ @Test
+ public void testAllOutput()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND);
+ tool.assertOnCleanExit();
+ assertThat(tool.getStdout()).contains("Keyspace: system_schema");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"Keyspace:")).isGreaterThan(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tTable:")).isGreaterThan(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tPaths:")).isGreaterThan(1);
+ }
+
+ @Test
+ public void testSelectedKeyspace()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"system_traces");
+ tool.assertOnCleanExit();
+ assertThat(tool.getStdout()).contains("Keyspace: system_traces");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"Keyspace:")).isEqualTo(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tTable:")).isGreaterThan(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tPaths:")).isGreaterThan(1);
+ }
+
+ @Test
+ public void testSelectedMultipleKeyspaces()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"system_traces", "system_auth");
+ tool.assertOnCleanExit();
+ assertThat(tool.getStdout()).contains("Keyspace: system_traces");
+ assertThat(tool.getStdout()).contains("Keyspace: system_auth");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"Keyspace:")).isEqualTo(2);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tTable:")).isGreaterThan(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tPaths:")).isGreaterThan(1);
+ }
+
+ @Test
+ public void testSelectedTable()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"system_auth.roles");
+ tool.assertOnCleanExit();
+ assertThat(tool.getStdout()).contains("Keyspace: system_auth");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"Keyspace:")).isEqualTo(1);
+ assertThat(tool.getStdout()).contains("Table: roles");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tTable:")).isEqualTo(1);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tPaths:")).isEqualTo(1);
+ }
+
+ @Test
+ public void testSelectedMultipleTables()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"system_auth.roles", "system_auth.role_members");
+ tool.assertOnCleanExit();
+ assertThat(tool.getStdout()).contains("Keyspace: system_auth");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"Keyspace:")).isEqualTo(1);
+ assertThat(tool.getStdout()).contains("Table: roles");
+ assertThat(tool.getStdout()).contains("Table: role_members");
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tTable:")).isEqualTo(2);
+ assertThat(StringUtils.countMatches(tool.getStdout(),
"\tPaths:")).isEqualTo(2);
+ }
+
+ @Test
+ public void testFormatArgJson()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"--format", "json");
+ tool.assertOnCleanExit();
+ }
+
+ @Test
+ public void testFormatArgYaml()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"--format", "yaml");
+ tool.assertOnCleanExit();
+ }
+
+ @Test
+ public void testFormatArgBad()
+ {
+ ToolRunner.ToolResult tool = ToolRunner.invokeNodetool(SUBCOMMAND,
"--format", "bad");
+ assertThat(tool.getStdout()).contains("arguments for -F are yaml and
json only.");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
