This is an automated email from the ASF dual-hosted git repository.
jmckenzie pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0371493772 Warn on unknown directories found in system keyspace
directory rather than kill node during startup checks
0371493772 is described below
commit 037149377224c5d6854fa4a0cacf44139273bce3
Author: Josh McKenzie <[email protected]>
AuthorDate: Tue Jul 26 15:15:48 2022 -0400
Warn on unknown directories found in system keyspace directory rather than
kill node during startup checks
Patch by Jeff Jirsa; reviewed by Josh McKenzie, Sam Tunnicliffe, and Marcus
Eriksson for CASSANDRA-17777
Co-authored-by: Jeff Jirsa <[email protected]>
Co-authored-by: Josh McKenzie <[email protected]>
---
CHANGES.txt | 1 +
.../org/apache/cassandra/db/SystemKeyspace.java | 9 ++++++
.../org/apache/cassandra/service/StartupCheck.java | 2 +-
.../apache/cassandra/service/StartupChecks.java | 34 +++++++++++++++++++++-
.../paxos/uncommitted/PaxosStateTracker.java | 2 +-
.../cassandra/service/StartupChecksTest.java | 11 +++++--
6 files changed, 53 insertions(+), 6 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 78456e28ad..c802607f42 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.2
+ * Warn on unknown directories found in system keyspace directory rather than
kill node during startup checks (CASSANDRA-17777)
* Log duplicate rows sharing a partition key found in verify and scrub
(CASSANDRA-17789)
* Add separate thread pool for Secondary Index building so it doesn't block
compactions (CASSANDRA-17781)
* Added JMX call to getSSTableCountPerTWCSBucket for TWCS (CASSANDRA-17774)
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java
b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index a1013e7955..c8c21b5e20 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -150,6 +150,7 @@ public final class SystemKeyspace
public static final String BATCHES = "batches";
public static final String PAXOS = "paxos";
public static final String PAXOS_REPAIR_HISTORY = "paxos_repair_history";
+ public static final String PAXOS_REPAIR_STATE = "_paxos_repair_state";
public static final String BUILT_INDEXES = "IndexInfo";
public static final String LOCAL = "local";
public static final String PEERS_V2 = "peers_v2";
@@ -185,6 +186,14 @@ public final class SystemKeyspace
@Deprecated public static final String LEGACY_SIZE_ESTIMATES =
"size_estimates";
@Deprecated public static final String LEGACY_SSTABLE_ACTIVITY =
"sstable_activity";
+ // Names of all tables that could have been a part of a system keyspace.
Useful for pre-flight checks.
+ // For details, see CASSANDRA-17777
+ public static final Set<String> ALL_TABLE_NAMES = ImmutableSet.of(
+ BATCHES, PAXOS, PAXOS_REPAIR_HISTORY, PAXOS_REPAIR_STATE,
BUILT_INDEXES, LOCAL, PEERS_V2, PEER_EVENTS_V2,
+ COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES,
TABLE_ESTIMATES_TYPE_PRIMARY,
+ TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY, AVAILABLE_RANGES_V2,
TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS,
+ BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS,
LEGACY_PEERS, LEGACY_PEER_EVENTS,
+ LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES,
LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY);
public static final TableMetadata Batches =
parse(BATCHES,
diff --git a/src/java/org/apache/cassandra/service/StartupCheck.java
b/src/java/org/apache/cassandra/service/StartupCheck.java
index 331b381d2f..c3790e8e99 100644
--- a/src/java/org/apache/cassandra/service/StartupCheck.java
+++ b/src/java/org/apache/cassandra/service/StartupCheck.java
@@ -31,7 +31,7 @@ import
org.apache.cassandra.service.StartupChecks.StartupCheckType;
* misconfiguration of cluster_name in cassandra.yaml.
*
* The StartupChecks class manages a collection of these tests, which it
executes
- * right at the beginning of the server settup process.
+ * right at the beginning of the server setup process.
*/
public interface StartupCheck
{
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java
b/src/java/org/apache/cassandra/service/StartupChecks.java
index 2ab5381697..c313ac8f0c 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -165,7 +165,7 @@ public class StartupChecks
/**
* Run the configured tests and return a report detailing the results.
- * @throws org.apache.cassandra.exceptions.StartupException if any test
determines that the
+ * @throws StartupException if any test determines that the
* system is not in an valid state to startup
* @param options options to pass to respective checks for their
configration
*/
@@ -571,6 +571,38 @@ public class StartupChecks
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException
{
+ String[] nameParts =
dir.toFile().getCanonicalPath().split(java.io.File.separator);
+ if (nameParts.length >= 2)
+ {
+ String tablePart = nameParts[nameParts.length - 1];
+ String ksPart = nameParts[nameParts.length - 2];
+
+ if (tablePart.contains("-"))
+ tablePart = tablePart.split("-")[0];
+
+ // In very old versions of Cassandra, we wouldn't
necessarily delete sstables from dropped system tables
+ // which were removed in various major version
upgrades (e.g system.Versions in 1.2)
+ if
(ksPart.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) &&
!SystemKeyspace.ALL_TABLE_NAMES.contains(tablePart))
+ {
+ // We can have snapshots of our system tables or
snapshots created with a -t tag of "system" that would trigger
+ // this potential warning, so we warn more softly
in the case that it's probably a snapshot.
+ if
(dir.toFile().getCanonicalPath().contains("snapshot"))
+ {
+ logger.info("Found unknown system directory
{}.{} at {} that contains the word snapshot. " +
+ "This may be left over from a
previous version of Cassandra or may be normal. " +
+ " Consider removing after
inspection if determined to be unnecessary.",
+ ksPart, tablePart,
dir.toFile().getCanonicalPath());
+ }
+ else
+ {
+ logger.warn("Found unknown system directory
{}.{} at {} - this is likely left over from a previous " +
+ "version of Cassandra and should
be removed after inspection.",
+ ksPart, tablePart,
dir.toFile().getCanonicalPath());
+ }
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ }
+
String name = dir.getFileName().toString();
return (name.equals(Directories.SNAPSHOT_SUBDIR)
|| name.equals(Directories.BACKUPS_SUBDIR)
diff --git
a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
index d3594b3979..c869249350 100644
---
a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
+++
b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java
@@ -83,7 +83,7 @@ public class PaxosStateTracker
return Boolean.getBoolean(TRUNCATE_BALLOT_METADATA_PROP);
}
- private static final String DIRECTORY = "system/_paxos_repair_state";
+ private static final String DIRECTORY = "system/" +
SystemKeyspace.PAXOS_REPAIR_STATE;
private final PaxosUncommittedTracker uncommitted;
private final PaxosBallotTracker ballots;
diff --git a/test/unit/org/apache/cassandra/service/StartupChecksTest.java
b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
index 4a63fc1eb2..77c72530c4 100644
--- a/test/unit/org/apache/cassandra/service/StartupChecksTest.java
+++ b/test/unit/org/apache/cassandra/service/StartupChecksTest.java
@@ -36,13 +36,10 @@ import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.service.DataResurrectionCheck.Heartbeat;
import org.apache.cassandra.utils.Clock;
-import static java.time.Instant.ofEpochMilli;
import static java.util.Collections.singletonList;
import static org.apache.cassandra.io.util.FileUtils.createTempFile;
-import static org.apache.cassandra.io.util.FileUtils.write;
import static
org.apache.cassandra.service.DataResurrectionCheck.HEARTBEAT_FILE_CONFIG_PROPERTY;
import static
org.apache.cassandra.service.StartupChecks.StartupCheckType.check_data_resurrection;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -115,6 +112,14 @@ public class StartupChecksTest
Files.createDirectories(backupDir);
copyInvalidLegacySSTables(backupDir);
startupChecks.verify(options);
+
+ // and in the system directory as of CASSANDRA-17777
+ new File(backupDir).deleteRecursive();
+ File dataDir = new
File(DatabaseDescriptor.getAllDataFileLocations()[0]);
+ Path systemDir = Paths.get(dataDir.absolutePath(), "system",
"InvalidSystemDirectory");
+ Files.createDirectories(systemDir);
+ copyInvalidLegacySSTables(systemDir);
+ startupChecks.verify(options);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
