This is an automated email from the ASF dual-hosted git repository.
paulo pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 31aa17a2a3 List snapshots of dropped tables
31aa17a2a3 is described below
commit 31aa17a2a3b18bdda723123cad811f075287807d
Author: Paulo Motta <[email protected]>
AuthorDate: Thu Apr 28 19:03:03 2022 -0300
List snapshots of dropped tables
Patch by Paulo Motta; Reviewed by Stefan Miklosovic and Brandon Williams or
CASSANDRA-16843
---
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 4 +-
src/java/org/apache/cassandra/db/Directories.java | 18 +-
.../cassandra/db/SnapshotDetailsTabularData.java | 2 +-
.../db/commitlog/CommitLogSegmentManagerCDC.java | 4 +-
.../apache/cassandra/service/StorageService.java | 29 +--
.../cassandra/service/snapshot/SnapshotLoader.java | 137 ++++++++++++
.../cassandra/service/snapshot/TableSnapshot.java | 223 +++++++++++++++----
.../cassandra/utils/DirectorySizeCalculator.java | 8 +-
.../cassandra/distributed/test/SnapshotsTest.java | 40 ++++
.../test/microbench/DirectorySizerBench.java | 2 +-
.../apache/cassandra/db/ColumnFamilyStoreTest.java | 23 +-
.../org/apache/cassandra/db/DirectoriesTest.java | 9 +-
.../apache/cassandra/index/sasi/SASIIndexTest.java | 4 +-
.../service/snapshot/SnapshotLoaderTest.java | 236 +++++++++++++++++++++
.../service/snapshot/SnapshotManagerTest.java | 15 +-
.../service/snapshot/TableSnapshotTest.java | 72 ++++---
17 files changed, 707 insertions(+), 120 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 8ad715c1b9..726cfad750 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.1
+ * List snapshots of dropped tables (CASSANDRA-16843)
* Add information whether sstables are dropped to SchemaChangeListener
(CASSANDRA-17582)
* Add a pluggable memtable API (CEP-11 / CASSANDRA-17034)
* Save sstable id as string in activity table (CASSANDRA-17585)
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 2bdac2b0e4..d46b20421a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2060,8 +2060,8 @@ public class ColumnFamilyStore implements
ColumnFamilyStoreMBean, Memtable.Owner
snapshotDirs.add(ephemeralSnapshotMarker.parent().toAbsolute());
// marker may create empty snapshot dir
}
- TableSnapshot snapshot = new TableSnapshot(metadata.keyspace,
metadata.name, tag, manifest.createdAt,
- manifest.expiresAt,
snapshotDirs, directories::getTrueAllocatedSizeIn);
+ TableSnapshot snapshot = new TableSnapshot(metadata.keyspace,
metadata.name, metadata.id.asUUID(), tag,
+ manifest.createdAt,
manifest.expiresAt, snapshotDirs);
StorageService.instance.addSnapshot(snapshot);
return snapshot;
diff --git a/src/java/org/apache/cassandra/db/Directories.java
b/src/java/org/apache/cassandra/db/Directories.java
index 972ba6d3cd..8672377b75 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -556,7 +556,7 @@ public class Directories
return getSnapshotManifestFile(snapshotDir);
}
- protected static File getSnapshotManifestFile(File snapshotDir)
+ public static File getSnapshotManifestFile(File snapshotDir)
{
return new File(snapshotDir, "manifest.json");
}
@@ -564,6 +564,11 @@ public class Directories
public File getSnapshotSchemaFile(String snapshotName)
{
File snapshotDir = getSnapshotDirectory(getDirectoryForNewSSTables(),
snapshotName);
+ return getSnapshotSchemaFile(snapshotDir);
+ }
+
+ public static File getSnapshotSchemaFile(File snapshotDir)
+ {
return new File(snapshotDir, "schema.cql");
}
@@ -981,8 +986,8 @@ public class Directories
protected TableSnapshot buildSnapshot(String tag, SnapshotManifest
manifest, Set<File> snapshotDirs) {
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
- return new TableSnapshot(metadata.keyspace, metadata.name, tag,
createdAt, expiresAt, snapshotDirs,
- this::getTrueAllocatedSizeIn);
+ return new TableSnapshot(metadata.keyspace, metadata.name,
metadata.id.asUUID(), tag, createdAt, expiresAt,
+ snapshotDirs);
}
@VisibleForTesting
@@ -1155,7 +1160,7 @@ public class Directories
if (!snapshotDir.isDirectory())
return 0;
- SSTableSizeSummer visitor = new SSTableSizeSummer(snapshotDir,
sstableLister(OnTxnErr.THROW).listFiles());
+ SSTableSizeSummer visitor = new
SSTableSizeSummer(sstableLister(OnTxnErr.THROW).listFiles());
try
{
Files.walkFileTree(snapshotDir.toPath(), visitor);
@@ -1244,10 +1249,9 @@ public class Directories
private class SSTableSizeSummer extends DirectorySizeCalculator
{
private final Set<String> toSkip;
- SSTableSizeSummer(File path, List<File> files)
+ SSTableSizeSummer(List<File> files)
{
- super(path);
- toSkip = files.stream().map(f ->
f.name()).collect(Collectors.toSet());
+ toSkip =
files.stream().map(File::name).collect(Collectors.toSet());
}
@Override
diff --git a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
index 73a4876df8..4e6ab116c4 100644
--- a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
+++ b/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
@@ -78,7 +78,7 @@ public class SnapshotDetailsTabularData
String createdAt = safeToString(details.getCreatedAt());
String expiresAt = safeToString(details.getExpiresAt());
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
- new Object[]{ details.getTag(), details.getKeyspace(),
details.getTable(), liveSize, totalSize, createdAt, expiresAt }));
+ new Object[]{ details.getTag(), details.getKeyspaceName(),
details.getTableName(), liveSize, totalSize, createdAt, expiresAt }));
}
catch (OpenDataException e)
{
diff --git
a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
index f00bbfc613..c75c3a9313 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDC.java
@@ -274,9 +274,11 @@ public class CommitLogSegmentManagerCDC extends
AbstractCommitLogSegmentManager
// track the total size between two dictionary size calculations
private final AtomicLong sizeInProgress;
+ private final File path;
+
CDCSizeTracker(CommitLogSegmentManagerCDC segmentManager, File path)
{
- super(path);
+ this.path = path;
this.segmentManager = segmentManager;
this.sizeInProgress = new AtomicLong(0);
}
diff --git a/src/java/org/apache/cassandra/service/StorageService.java
b/src/java/org/apache/cassandra/service/StorageService.java
index 97d5204722..dd35c70fdd 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -73,6 +73,7 @@ import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
+import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -4135,24 +4136,26 @@ public class StorageService extends
NotificationBroadcasterSupport implements IE
public Map<String, TabularData> getSnapshotDetails(Map<String, String>
options)
{
+ boolean skipExpiring = options != null &&
Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
+
+ SnapshotLoader loader = new SnapshotLoader();
Map<String, TabularData> snapshotMap = new HashMap<>();
- for (Keyspace keyspace : Keyspace.all())
+
+ for (TableSnapshot snapshot : loader.loadSnapshots())
{
- for (ColumnFamilyStore cfStore : keyspace.getColumnFamilyStores())
- {
- for (Map.Entry<String, TableSnapshot> snapshotDetail :
TableSnapshot.filter(cfStore.listSnapshots(), options).entrySet())
- {
- TabularDataSupport data = (TabularDataSupport)
snapshotMap.get(snapshotDetail.getKey());
- if (data == null)
- {
- data = new
TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE);
- snapshotMap.put(snapshotDetail.getKey(), data);
- }
+ if (skipExpiring && snapshot.isExpiring())
+ continue;
- SnapshotDetailsTabularData.from(snapshotDetail.getValue(),
data);
- }
+ TabularDataSupport data = (TabularDataSupport)
snapshotMap.get(snapshot.getTag());
+ if (data == null)
+ {
+ data = new
TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE);
+ snapshotMap.put(snapshot.getTag(), data);
}
+
+ SnapshotDetailsTabularData.from(snapshot, data);
}
+
return snapshotMap;
}
diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
new file mode 100644
index 0000000000..75ae9ee5b5
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.snapshot;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.util.File;
+
+import static org.apache.cassandra.db.Directories.SNAPSHOT_SUBDIR;
+import static
org.apache.cassandra.service.snapshot.TableSnapshot.buildSnapshotId;
+
+/**
+ * Loads snapshot metadata from data directories
+ */
+public class SnapshotLoader extends SimpleFileVisitor<Path>
+{
+ private static final Logger logger =
LoggerFactory.getLogger(SnapshotLoader.class);
+
+ static final Pattern SNAPSHOT_DIR_PATTERN =
Pattern.compile("(?<keyspace>\\w+)/(?<tableName>\\w+)\\-(?<tableId>[0-9a-f]{32})/snapshots/(?<tag>[\\w-]+)$");
+
+ private final Collection<Path> dataDirectories;
+ private final Map<String, TableSnapshot.Builder> snapshots = new
HashMap<>();
+
+ public SnapshotLoader()
+ {
+ this(DatabaseDescriptor.getAllDataFileLocations());
+ }
+
+ public SnapshotLoader(String[] dataDirectories)
+ {
+ this.dataDirectories =
Arrays.stream(dataDirectories).map(Paths::get).collect(Collectors.toList());
+ }
+
+ public SnapshotLoader(Collection<Path> dataDirs)
+ {
+ this.dataDirectories = dataDirs;
+ }
+
+ public Set<TableSnapshot> loadSnapshots()
+ {
+ for (Path dataDir : dataDirectories)
+ {
+ try
+ {
+ Files.walkFileTree(dataDir, Collections.EMPTY_SET, 5, this);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(String.format("Error while loading
snapshots from %s", dataDir));
+ }
+ }
+ return
snapshots.values().stream().map(TableSnapshot.Builder::build).collect(Collectors.toSet());
+ }
+
+ public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes
attrs)
+ {
+ if
(subdir.getParent().getFileName().toString().equals(SNAPSHOT_SUBDIR))
+ {
+ logger.trace("Processing directory " + subdir);
+ Matcher snapshotDirMatcher =
SNAPSHOT_DIR_PATTERN.matcher(subdir.toString());
+ if (snapshotDirMatcher.find())
+ {
+ try
+ {
+ loadSnapshotFromDir(snapshotDirMatcher, subdir);
+ } catch (Throwable e)
+ {
+ logger.warn("Could not load snapshot from {}.", subdir, e);
+ }
+ }
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+
+ return subdir.getFileName().equals(Directories.BACKUPS_SUBDIR)
+ ? FileVisitResult.SKIP_SUBTREE
+ : FileVisitResult.CONTINUE;
+ }
+
+ private void loadSnapshotFromDir(Matcher snapshotDirMatcher, Path
snapshotDir)
+ {
+ String keyspaceName = snapshotDirMatcher.group("keyspace");
+ String tableName = snapshotDirMatcher.group("tableName");
+ UUID tableId = parseUUID(snapshotDirMatcher.group("tableId"));
+ String tag = snapshotDirMatcher.group("tag");
+ String snapshotId = buildSnapshotId(keyspaceName, tableName, tableId,
tag);
+ TableSnapshot.Builder builder = snapshots.computeIfAbsent(snapshotId,
k -> new TableSnapshot.Builder(keyspaceName, tableName, tableId, tag));
+ builder.addSnapshotDir(new File(snapshotDir));
+ }
+
+ /**
+ * Given an UUID string without dashes (ie.
c7e513243f0711ec9bbc0242ac130002)
+ * return an UUID object (ie. c7e51324-3f07-11ec-9bbc-0242ac130002)
+ */
+ protected static UUID parseUUID(String uuidWithoutDashes) throws
IllegalArgumentException
+ {
+ assert uuidWithoutDashes.length() == 32 &&
!uuidWithoutDashes.contains("-");
+ String dashedUUID =
uuidWithoutDashes.replaceFirst("([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]+)",
"$1-$2-$3-$4-$5");
+ return UUID.fromString(dashedUUID);
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
index 185cd45c4b..7a2cdc0f89 100644
--- a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
+++ b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
@@ -17,50 +17,73 @@
*/
package org.apache.cassandra.service.snapshot;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.time.Instant;
import java.util.Collection;
-import java.util.Map;
+import java.util.HashSet;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
+import java.util.UUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.DirectorySizeCalculator;
public class TableSnapshot
{
- private final String keyspace;
- private final String table;
+ private static final Logger logger =
LoggerFactory.getLogger(TableSnapshot.class);
+
+ private final String keyspaceName;
+ private final String tableName;
+ private final UUID tableId;
private final String tag;
private final Instant createdAt;
private final Instant expiresAt;
private final Set<File> snapshotDirs;
- private final Function<File, Long> trueDiskSizeComputer;
- public TableSnapshot(String keyspace, String table, String tag, Instant
createdAt,
- Instant expiresAt, Set<File> snapshotDirs,
- Function<File, Long> trueDiskSizeComputer)
+ public TableSnapshot(String keyspaceName, String tableName, UUID tableId,
+ String tag, Instant createdAt, Instant expiresAt,
+ Set<File> snapshotDirs)
{
- this.keyspace = keyspace;
- this.table = table;
+ this.keyspaceName = keyspaceName;
+ this.tableName = tableName;
+ this.tableId = tableId;
this.tag = tag;
this.createdAt = createdAt;
this.expiresAt = expiresAt;
this.snapshotDirs = snapshotDirs;
- this.trueDiskSizeComputer = trueDiskSizeComputer;
}
- public String getKeyspace()
+ /**
+ * Unique identifier of a snapshot. Used
+ * only to deduplicate snapshots internally,
+ * not exposed externally.
+ *
+ * Format: "$ks:$table_name:$table_id:$tag"
+ */
+ public String getId()
+ {
+ return buildSnapshotId(keyspaceName, tableName, tableId, tag);
+ }
+
+ public String getKeyspaceName()
{
- return keyspace;
+ return keyspaceName;
}
- public String getTable()
+ public String getTableName()
{
- return table;
+ return tableName;
}
public String getTag()
@@ -113,7 +136,21 @@ public class TableSnapshot
public long computeTrueSizeBytes()
{
- return
snapshotDirs.stream().mapToLong(trueDiskSizeComputer::apply).sum();
+ DirectorySizeCalculator visitor = new SnapshotTrueSizeCalculator();
+
+ for (File snapshotDir : snapshotDirs)
+ {
+ try
+ {
+ Files.walkFileTree(snapshotDir.toPath(), visitor);
+ }
+ catch (IOException e)
+ {
+ logger.error("Could not calculate the size of {}.",
snapshotDir, e);
+ }
+ }
+
+ return visitor.getAllocatedSize();
}
public Collection<File> getDirectories()
@@ -121,17 +158,30 @@ public class TableSnapshot
return snapshotDirs;
}
- @Override
- public String toString()
+ public Optional<File> getManifestFile()
{
- return "TableSnapshot{" +
- "keyspace='" + keyspace + '\'' +
- ", table='" + table + '\'' +
- ", tag='" + tag + '\'' +
- ", createdAt=" + createdAt +
- ", expiresAt=" + expiresAt +
- ", snapshotDirs=" + snapshotDirs +
- '}';
+ for (File snapshotDir : snapshotDirs)
+ {
+ File manifestFile =
Directories.getSnapshotManifestFile(snapshotDir);
+ if (manifestFile.exists())
+ {
+ return Optional.of(manifestFile);
+ }
+ }
+ return Optional.empty();
+ }
+
+ public Optional<File> getSchemaFile()
+ {
+ for (File snapshotDir : snapshotDirs)
+ {
+ File schemaFile = Directories.getSnapshotSchemaFile(snapshotDir);
+ if (schemaFile.exists())
+ {
+ return Optional.of(schemaFile);
+ }
+ }
+ return Optional.empty();
}
@Override
@@ -139,28 +189,121 @@ public class TableSnapshot
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
- TableSnapshot that = (TableSnapshot) o;
- return Objects.equals(keyspace, that.keyspace) &&
Objects.equals(table, that.table)
- && Objects.equals(tag, that.tag) && Objects.equals(createdAt,
that.createdAt)
- && Objects.equals(expiresAt, that.expiresAt) &&
Objects.equals(snapshotDirs, that.snapshotDirs);
+ TableSnapshot snapshot = (TableSnapshot) o;
+ return Objects.equals(keyspaceName, snapshot.keyspaceName) &&
Objects.equals(tableName, snapshot.tableName) &&
+ Objects.equals(tableId, snapshot.tableId) &&
Objects.equals(tag, snapshot.tag) &&
+ Objects.equals(createdAt, snapshot.createdAt) &&
Objects.equals(expiresAt, snapshot.expiresAt) &&
+ Objects.equals(snapshotDirs, snapshot.snapshotDirs);
}
@Override
public int hashCode()
{
- return Objects.hash(keyspace, table, tag, createdAt, expiresAt,
snapshotDirs);
+ return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt,
expiresAt, snapshotDirs);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TableSnapshot{" +
+ "keyspaceName='" + keyspaceName + '\'' +
+ ", tableName='" + tableName + '\'' +
+ ", tableId=" + tableId +
+ ", tag='" + tag + '\'' +
+ ", createdAt=" + createdAt +
+ ", expiresAt=" + expiresAt +
+ ", snapshotDirs=" + snapshotDirs +
+ '}';
+ }
+
+ static class Builder {
+ private final String keyspaceName;
+ private final String tableName;
+ private final UUID tableId;
+ private final String tag;
+
+ private Instant createdAt = null;
+ private Instant expiresAt = null;
+
+ private final Set<File> snapshotDirs = new HashSet<>();
+
+ Builder(String keyspaceName, String tableName, UUID tableId, String
tag)
+ {
+ this.keyspaceName = keyspaceName;
+ this.tableName = tableName;
+ this.tag = tag;
+ this.tableId = tableId;
+ }
+
+ void addSnapshotDir(File snapshotDir)
+ {
+ snapshotDirs.add(snapshotDir);
+ File manifestFile = new File(snapshotDir, "manifest.json");
+ if (manifestFile.exists() && createdAt == null && expiresAt ==
null) {
+ loadTimestampsFromManifest(manifestFile);
+ }
+ }
+
+ private void loadTimestampsFromManifest(File manifestFile)
+ {
+ try
+ {
+ logger.debug("Loading snapshot manifest from {}",
manifestFile);
+ SnapshotManifest manifest =
SnapshotManifest.deserializeFromJsonFile(manifestFile);
+ createdAt = manifest.createdAt;
+ expiresAt = manifest.expiresAt;
+ }
+ catch (IOException e)
+ {
+ logger.warn("Cannot read manifest file {} of snapshot {}.",
manifestFile, tag, e);
+ }
+ }
+
+ TableSnapshot build()
+ {
+ return new TableSnapshot(keyspaceName, tableName, tableId, tag,
createdAt, expiresAt, snapshotDirs);
+ }
}
- public static Map<String, TableSnapshot> filter(Map<String, TableSnapshot>
snapshots, Map<String, String> options)
+ protected static String buildSnapshotId(String keyspaceName, String
tableName, UUID tableId, String tag)
{
- if (options == null)
- return snapshots;
+ return String.format("%s:%s:%s:%s", keyspaceName, tableName, tableId,
tag);
+ }
- boolean skipExpiring =
Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
+ public static class SnapshotTrueSizeCalculator extends
DirectorySizeCalculator
+ {
+ /**
+ * Snapshots are composed of hard-linked sstables. The true snapshot
size should only include
+ * snapshot files which do not contain a corresponding "live" sstable
file.
+ */
+ @Override
+ public boolean isAcceptable(Path snapshotFilePath)
+ {
+ return !getLiveFileFromSnapshotFile(snapshotFilePath).exists();
+ }
+ }
- return snapshots.entrySet()
- .stream()
- .filter(entry -> !skipExpiring ||
!entry.getValue().isExpiring())
- .collect(Collectors.toMap(Map.Entry::getKey,
Map.Entry::getValue));
+ /**
+ * Returns the corresponding live file for a given snapshot file.
+ *
+ * Example:
+ * - Base table:
+ * - Snapshot file:
~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db
+ * - Live file:
~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db
+ * - Secondary index:
+ * - Snapshot file:
~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db
+ * - Live file:
~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db
+ *
+ */
+ static File getLiveFileFromSnapshotFile(Path snapshotFilePath)
+ {
+ // Snapshot directory structure format is
{data_dir}/snapshots/{snapshot_name}/{snapshot_file}
+ Path liveDir = snapshotFilePath.getParent().getParent().getParent();
+ if (Directories.isSecondaryIndexFolder(new
File(snapshotFilePath.getParent().toFile())))
+ {
+ // Snapshot file structure format is
{data_dir}/snapshots/{snapshot_name}/.{index}/{sstable-component}.db
+ liveDir = Paths.get(liveDir.getParent().toString(),
snapshotFilePath.getParent().getFileName().toString());
+ }
+ return new File(liveDir.toString(),
snapshotFilePath.getFileName().toString());
}
}
diff --git a/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java
b/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java
index f0cfdea021..94f9b229b4 100644
--- a/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java
+++ b/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java
@@ -27,17 +27,13 @@ import java.nio.file.attribute.BasicFileAttributes;
/**
* Walks directory recursively, summing up total contents of files within.
*/
-import org.apache.cassandra.io.util.File;
-
public class DirectorySizeCalculator extends SimpleFileVisitor<Path>
{
private volatile long size = 0;
- protected final File path;
- public DirectorySizeCalculator(File path)
+ public DirectorySizeCalculator()
{
super();
- this.path = path;
}
public boolean isAcceptable(Path file)
@@ -54,7 +50,7 @@ public class DirectorySizeCalculator extends
SimpleFileVisitor<Path>
}
@Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws
IOException
+ public FileVisitResult visitFileFailed(Path file, IOException exc)
{
return FileVisitResult.CONTINUE;
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java
b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java
index c56adfe7b1..1029301fca 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.util.Arrays;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -52,6 +53,14 @@ public class SnapshotsTest extends TestBaseImpl
cluster = init(Cluster.build(1).withConfig(c ->
c.with(Feature.GOSSIP)).start());
}
+
+ @After
+ public void clearAllSnapshots()
+ {
+ cluster.schemaChange("DROP KEYSPACE IF EXISTS default;");
+ cluster.get(1).nodetoolResult("clearsnapshot",
"--all").asserts().success();
+ }
+
@AfterClass
public static void after()
{
@@ -182,6 +191,37 @@ public class SnapshotsTest extends TestBaseImpl
listSnapshotsResult.stdoutNotContains("first");
}
+ @Test
+ public void testListSnapshotOfDroppedTable()
+ {
+ IInvokableInstance instance = cluster.get(1);
+
+ cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS default WITH
replication = {'class': 'SimpleStrategy', 'replication_factor': 2};");
+ cluster.schemaChange("CREATE TABLE default.tbl (key int, value text,
PRIMARY KEY (key))");
+
+ populate(cluster);
+
+ instance.nodetoolResult("snapshot",
+ "-t", "tag1",
+ "-kt",
"default.tbl").asserts().success();
+
+ // Check snapshot is listed when table is not dropped
+
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains("tag1");
+
+ // Drop Table
+ cluster.schemaChange("DROP TABLE default.tbl;");
+
+ // Check snapshot is listed after table is dropped
+
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains("tag1");
+
+ // Restart node
+ stopUnchecked(instance);
+ instance.startup();
+
+ // Check snapshot of dropped table still exists after restart
+
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains("tag1");
+ }
+
@Test
public void testSameTimestampOnEachTableOfSnaphot()
{
diff --git
a/test/microbench/org/apache/cassandra/test/microbench/DirectorySizerBench.java
b/test/microbench/org/apache/cassandra/test/microbench/DirectorySizerBench.java
index c4466e11b0..ad72f3d02c 100644
---
a/test/microbench/org/apache/cassandra/test/microbench/DirectorySizerBench.java
+++
b/test/microbench/org/apache/cassandra/test/microbench/DirectorySizerBench.java
@@ -66,7 +66,7 @@ public class DirectorySizerBench
// Test w/25,600 files, 100x the load of a full default CommitLog
(8192) divided by size (32 per)
populateRandomFiles(tempDir, 25600);
- sizer = new DirectorySizeCalculator(tempDir);
+ sizer = new DirectorySizeCalculator();
}
@TearDown
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 633328d4c8..6d8bd0fe05 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -28,8 +28,10 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.junit.Assert;
import org.junit.Before;
@@ -261,7 +263,7 @@ public class ColumnFamilyStoreTest
}
@Test
- public void testSnapshotSize()
+ public void testSnapshotSize() throws IOException
{
// cleanup any previous test gargbage
ColumnFamilyStore cfs =
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
@@ -286,7 +288,7 @@ public class ColumnFamilyStoreTest
// check that sizeOnDisk > trueSize = 0
TableSnapshot details = snapshotDetails.get("basic");
assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes());
- assertThat(details.computeTrueSizeBytes()).isZero();
+
assertThat(details.computeTrueSizeBytes()).isEqualTo(getSnapshotManifestAndSchemaFileSizes(details));
// compact base table to make trueSize > 0
cfs.forceMajorCompaction();
@@ -296,8 +298,7 @@ public class ColumnFamilyStoreTest
// Check that truesize now is > 0
snapshotDetails = cfs.listSnapshots();
details = snapshotDetails.get("basic");
-
assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes());
- assertThat(details.computeTrueSizeBytes()).isPositive();
+
assertThat(details.computeSizeOnDiskBytes()).isEqualTo(details.computeTrueSizeBytes());
}
@Test
@@ -609,4 +610,18 @@ public class ColumnFamilyStoreTest
assertEquals(0, ssTableFiles.size());
cfs.clearUnsafe();
}
+
+ @VisibleForTesting
+ public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot
snapshot) throws IOException
+ {
+ Optional<File> schemaFile = snapshot.getSchemaFile();
+ Optional<File> manifestFile = snapshot.getManifestFile();
+
+ long schemaAndManifestFileSizes = 0;
+
+ schemaAndManifestFileSizes += schemaFile.isPresent() ?
schemaFile.get().length() : 0;
+ schemaAndManifestFileSizes += manifestFile.isPresent() ?
manifestFile.get().length() : 0;
+
+ return schemaAndManifestFileSizes;
+ }
}
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index eb6a8b6cc3..3779eccbec 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -199,7 +199,7 @@ public class DirectoriesTest
{
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
- return new TableSnapshot(table.keyspace, table.name, tag,
createdAt, expiresAt, Collections.singleton(snapshotDir), null);
+ return new TableSnapshot(table.keyspace, table.name,
table.id.asUUID(), tag, createdAt, expiresAt,
Collections.singleton(snapshotDir));
}
}
@@ -417,11 +417,8 @@ public class DirectoriesTest
// check snapshot details
Map<String, TableSnapshot> parentSnapshotDetail =
parentDirectories.listSnapshots();
assertTrue(parentSnapshotDetail.containsKey("test"));
- assertEquals(30L,
parentSnapshotDetail.get("test").computeTrueSizeBytes());
-
- Map<String, TableSnapshot> indexSnapshotDetail =
indexDirectories.listSnapshots();
- assertTrue(indexSnapshotDetail.containsKey("test"));
- assertEquals(40L,
indexSnapshotDetail.get("test").computeTrueSizeBytes());
+ // CASSANDRA-17357: include indexes when computing true size of parent
table
+ assertEquals(70L,
parentSnapshotDetail.get("test").computeTrueSizeBytes());
// check backup directory
File parentBackupDirectory =
Directories.getBackupsDirectory(parentDesc);
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 8c223a5a7f..441d7b00ff 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -101,6 +101,7 @@ import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.*;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static
org.apache.cassandra.db.ColumnFamilyStoreTest.getSnapshotManifestAndSchemaFileSizes;
public class SASIIndexTest
{
@@ -211,7 +212,8 @@ public class SASIIndexTest
TableSnapshot details = store.listSnapshots().get(snapshotName);
// check that SASI components are included in the computation of
snapshot size
- Assert.assertEquals(tableSize + indexSize,
details.computeTrueSizeBytes());
+ long snapshotSize = tableSize + indexSize +
getSnapshotManifestAndSchemaFileSizes(details);
+ Assert.assertEquals(snapshotSize, details.computeTrueSizeBytes());
}
finally
{
diff --git
a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
new file mode 100644
index 0000000000..27f294e8d5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.snapshot;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.util.File;
+import org.assertj.core.util.Lists;
+
+import static
org.apache.cassandra.service.snapshot.SnapshotLoader.SNAPSHOT_DIR_PATTERN;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SnapshotLoaderTest
+{
+ static String DATA_DIR_1 = "data1";
+ static String DATA_DIR_2 = "data2";
+ static String DATA_DIR_3 = "data3";
+ static String[] DATA_DIRS = new String[]{DATA_DIR_1, DATA_DIR_2,
DATA_DIR_3};
+
+ static String KEYSPACE_1 = "ks1";
+ static String TABLE1_NAME = "table_1";
+ static UUID TABLE1_ID = UUID.randomUUID();
+ static String TABLE2_NAME = "table2";
+ static UUID TABLE2_ID = UUID.randomUUID();
+
+ static String KEYSPACE_2 = "ks2";
+ static String TABLE3_NAME = "table_3";
+ static UUID TABLE3_ID = UUID.randomUUID();
+
+ static String TAG1 = "tag1";
+ static String TAG2 = "tag2";
+ static String TAG3 = "tag3";
+
+ static String INVALID_NAME = "#test#";
+ static String INVALID_ID = "XPTO";
+
+ @ClassRule
+ public static TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Test
+ public void testMatcher()
+ {
+ String INDEX_SNAPSHOT =
"/user/.ccm/test/node1/data0/ks/indexed_table-24b241e0c58f11eca526336fc2c671ab/snapshots/test";
+
assertThat(SNAPSHOT_DIR_PATTERN.matcher(INDEX_SNAPSHOT).find()).isTrue();
+
+ String TABLE_SNAPSHOT =
"/Users/pmottagomes/.ccm/test/node1/data0/ks/my_table-1a025b40c58f11eca526336fc2c671ab/snapshots/test";
+
assertThat(SNAPSHOT_DIR_PATTERN.matcher(TABLE_SNAPSHOT).find()).isTrue();
+
+ String DROPPED_SNAPSHOT =
"/Users/pmottagomes/.ccm/test/node1/data0/ks/my_table-e5c58330c58d11eca526336fc2c671ab/snapshots/dropped-1650997415751-my_table";
+
assertThat(SNAPSHOT_DIR_PATTERN.matcher(DROPPED_SNAPSHOT).find()).isTrue();
+ }
+
+ @Test
+ public void testNoSnapshots() throws IOException
+ {
+ // Create table directories on all data directories without snapshots
+ File baseDir = new File(tmpDir.newFolder());
+ for (String dataDir : DATA_DIRS)
+ {
+ createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME,
TABLE1_ID));
+ createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE2_NAME,
TABLE2_ID));
+ createDir(baseDir, dataDir, KEYSPACE_2, tableDirName(TABLE3_NAME,
TABLE3_ID));
+ }
+
+ // Check no snapshots are found
+ SnapshotLoader loader = new
SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+
Paths.get(baseDir.toString(), DATA_DIR_2),
+
Paths.get(baseDir.toString(), DATA_DIR_3)));
+ assertThat(loader.loadSnapshots()).isEmpty();
+ }
+
+ @Test
+ public void testSnapshotsWithoutManifests() throws IOException
+ {
+ Set<File> tag1Files = new HashSet<>();
+ Set<File> tag2Files = new HashSet<>();
+ Set<File> tag3Files = new HashSet<>();
+
+ // Create one snapshot per table - without manifests:
+ // - ks1.t1 : tag1
+ // - ks1.t2 : tag2
+ // - ks2.t3 : tag3
+ File baseDir = new File(tmpDir.newFolder());
+ for (String dataDir : DATA_DIRS)
+ {
+ tag1Files.add(createDir(baseDir, dataDir, KEYSPACE_1,
tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1));
+ tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1,
tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2));
+ tag3Files.add(createDir(baseDir, dataDir, KEYSPACE_2,
tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3));
+ }
+
+ // Verify all 3 snapshots are found correctly from data directories
+ SnapshotLoader loader = new
SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+
Paths.get(baseDir.toString(), DATA_DIR_2),
+
Paths.get(baseDir.toString(), DATA_DIR_3)));
+ Set<TableSnapshot> snapshots = loader.loadSnapshots();
+ assertThat(snapshots).hasSize(3);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1,
TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1,
TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2,
TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files));
+ }
+
+ @Test
+ public void testSnapshotsWithManifests() throws IOException
+ {
+ Set<File> tag1Files = new HashSet<>();
+ Set<File> tag2Files = new HashSet<>();
+ Set<File> tag3Files = new HashSet<>();
+
+ // Create one snapshot per table:
+ // - ks1.t1 : tag1
+ // - ks1.t2 : tag2
+ // - ks2.t3 : tag3
+ File baseDir = new File(tmpDir.newFolder());
+ for (String dataDir : DATA_DIRS)
+ {
+ tag1Files.add(createDir(baseDir, dataDir, KEYSPACE_1,
tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1));
+ tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1,
tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2));
+ tag3Files.add(createDir(baseDir, dataDir, KEYSPACE_2,
tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3));
+ }
+
+ // Write manifest for snapshot tag1 on random location
+ Instant tag1Ts = Instant.now();
+ File tag1ManifestLocation = tag1Files.toArray(new
File[0])[ThreadLocalRandom.current().nextInt(tag1Files.size())];
+ writeManifest(tag1ManifestLocation, tag1Ts, null);
+
+ // Write manifest for snapshot tag2 on random location
+ Instant tag2Ts = Instant.now().plusSeconds(10);
+ DurationSpec tag2Ttl = new DurationSpec("10h");
+ File tag2ManifestLocation = tag2Files.toArray(new
File[0])[ThreadLocalRandom.current().nextInt(tag2Files.size())];
+ writeManifest(tag2ManifestLocation, tag2Ts, tag2Ttl);
+
+ // Write manifest for snapshot tag3 on random location
+ Instant tag3Ts = Instant.now().plusSeconds(20);
+ File tag3ManifestLocation = tag3Files.toArray(new
File[0])[ThreadLocalRandom.current().nextInt(tag3Files.size())];
+ writeManifest(tag3ManifestLocation, tag3Ts, null);
+
+ // Verify all 3 snapshots are found correctly from data directories
+ SnapshotLoader loader = new
SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+
Paths.get(baseDir.toString(), DATA_DIR_2),
+
Paths.get(baseDir.toString(), DATA_DIR_3)));
+ Set<TableSnapshot> snapshots = loader.loadSnapshots();
+ assertThat(snapshots).hasSize(3);
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1,
TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1,
TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()),
tag2Files));
+ assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2,
TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files));
+ }
+
+ @Test
+ public void testInvalidSnapshotsAreNotLoaded() throws IOException
+ {
+ Set<File> tag1Files = new HashSet<>();
+ Set<File> tag2Files = new HashSet<>();
+ Set<File> tag3Files = new HashSet<>();
+
+ // Create invalid snapshot directory structure
+ // - /data_dir/#test#/table1-validuuid/snapshot/tag1
+ // - /data_dir/ks1/#test#-validuuid/snapshot/tag2
+ // - /data_dir/ks2/table3-invaliduuid/snapshot/tag3
+ File baseDir = new File(tmpDir.newFolder());
+ for (String dataDir : DATA_DIRS)
+ {
+ tag1Files.add(createDir(baseDir, dataDir, INVALID_NAME,
tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1));
+ tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1,
tableDirName(INVALID_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2));
+ tag3Files.add(createDir(baseDir, dataDir, KEYSPACE_2,
String.format("%s-%s", TABLE3_NAME, INVALID_ID), Directories.SNAPSHOT_SUBDIR,
TAG3));
+ }
+
+ // Check no snapshots are loaded
+ SnapshotLoader loader = new
SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
+
Paths.get(baseDir.toString(), DATA_DIR_2),
+
Paths.get(baseDir.toString(), DATA_DIR_3)));
+ assertThat(loader.loadSnapshots()).isEmpty();
+ }
+
+ @Test
+ public void testParseUUID()
+ {
+
assertThat(SnapshotLoader.parseUUID("c7e513243f0711ec9bbc0242ac130002")).isEqualTo(UUID.fromString("c7e51324-3f07-11ec-9bbc-0242ac130002"));
+ }
+
+ private void writeManifest(File snapshotDir, Instant creationTime,
DurationSpec ttl) throws IOException
+ {
+ SnapshotManifest manifest = new
SnapshotManifest(Lists.newArrayList("f1", "f2", "f3"), ttl, creationTime);
+ manifest.serializeToJsonFile(getManifestFile(snapshotDir));
+ }
+
+ private static File createDir(File baseDir, String... subdirs)
+ {
+ File file = new File(Paths.get(baseDir.toString(),
subdirs).toString());
+ file.toJavaIOFile().mkdirs();
+ return file;
+ }
+
+ static String tableDirName(String tableName, UUID tableId)
+ {
+ return String.format("%s-%s", tableName, removeDashes(tableId));
+ }
+
+ static String removeDashes(UUID id)
+ {
+ return id.toString().replace("-", "");
+ }
+
+ public static File getManifestFile(File snapshotDir)
+ {
+ return new File(snapshotDir, "manifest.json");
+ }
+}
diff --git
a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
index 9530c95e61..b07bb23487 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.service.snapshot;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
+import java.util.UUID;
import java.util.stream.Stream;
import org.junit.BeforeClass;
@@ -52,13 +53,13 @@ public class SnapshotManagerTest
private TableSnapshot generateSnapshotDetails(String tag, Instant
expiration) throws Exception {
return new TableSnapshot(
- "ks",
- "tbl",
- tag,
- Instant.EPOCH,
- expiration,
- createFolders(temporaryFolder),
- (file) -> 0L
+ "ks",
+ "tbl",
+ UUID.randomUUID(),
+ tag,
+ Instant.EPOCH,
+ expiration,
+ createFolders(temporaryFolder)
);
}
diff --git
a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
index 0b77472e71..4bb1756c31 100644
--- a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
+++ b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
@@ -19,10 +19,12 @@
package org.apache.cassandra.service.snapshot;
import java.io.IOException;
+import java.nio.file.Paths;
import java.time.Instant;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.UUID;
import org.junit.Before;
import org.junit.ClassRule;
@@ -68,12 +70,11 @@ public class TableSnapshotTest
TableSnapshot snapshot = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
null,
null,
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(snapshot.exists()).isTrue();
@@ -90,12 +91,11 @@ public class TableSnapshotTest
TableSnapshot snapshot = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
null,
null,
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -103,12 +103,11 @@ public class TableSnapshotTest
snapshot = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
now(),
null,
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -116,12 +115,11 @@ public class TableSnapshotTest
snapshot = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
now(),
now().plusSeconds(1000),
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isFalse();
@@ -129,12 +127,11 @@ public class TableSnapshotTest
snapshot = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
now(),
now().minusSeconds(1000),
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isTrue();
@@ -158,14 +155,11 @@ public class TableSnapshotTest
TableSnapshot tableDetails = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
null,
null,
- folders,
- (File file) -> {
- return 0L;
- }
- );
+ folders);
Long res = 0L;
@@ -187,19 +181,19 @@ public class TableSnapshotTest
TableSnapshot tableDetails = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some",
null,
null,
- folders,
- File::length
- );
+ folders);
Long res = 0L;
for (File dir : folders)
{
- writeBatchToFile(new File(dir, "tmp"));
- res += dir.length();
+ File file = new File(dir, "tmp");
+ writeBatchToFile(file);
+ res += file.length();
}
assertThat(tableDetails.computeTrueSizeBytes()).isGreaterThan(0L);
@@ -216,25 +210,41 @@ public class TableSnapshotTest
TableSnapshot withCreatedAt = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some1",
createdAt,
null,
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(withCreatedAt.getCreatedAt()).isEqualTo(createdAt);
// When createdAt is null, it should return the snapshot folder
minimum update time
TableSnapshot withoutCreatedAt = new TableSnapshot(
"ks",
"tbl",
+ UUID.randomUUID(),
"some1",
null,
null,
- folders,
- (File file) -> 0L
- );
+ folders);
assertThat(withoutCreatedAt.getCreatedAt()).isEqualTo(Instant.ofEpochMilli(folders.stream().mapToLong(f
-> f.lastModified()).min().getAsLong()));
}
+ @Test
+ public void testGetLiveFileFromSnapshotFile()
+ {
+
testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db",
+
"~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db");
+ }
+
+ @Test
+ public void testGetLiveFileFromSnapshotIndexFile()
+ {
+
testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db",
+
"~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db");
+ }
+
+ public void testGetLiveFileFromSnapshotFile(String snapshotFile, String
expectedLiveFile)
+ {
+
assertThat(TableSnapshot.getLiveFileFromSnapshotFile(Paths.get(snapshotFile)).toString()).isEqualTo(expectedLiveFile);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
