merge from 1.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af1dc0f5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af1dc0f5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af1dc0f5 Branch: refs/heads/trunk Commit: af1dc0f56a560ddee1e98bfea677bf534d38cc39 Parents: 249456b c1148ce Author: Jonathan Ellis <[email protected]> Authored: Thu May 3 18:31:29 2012 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Thu May 3 18:31:29 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 9 +++- src/java/org/apache/cassandra/db/Directories.java | 33 ++++---------- .../cassandra/db/compaction/CompactionTask.java | 20 ++------- .../apache/cassandra/io/sstable/Descriptor.java | 15 ++++++- .../cassandra/io/sstable/SSTableMetadata.java | 4 +- 5 files changed, 38 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index b13d69b,1c1c184..1b86758 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,121 -1,19 +1,128 @@@ -1.0.10 +1.1.1-dev + * include tombstone size in ColumnFamily.size, which can prevent OOM + during sudden mass delete operations (CASSANDRA-3741) + * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142) + * Optimize reads when row deletion timestamps allow us to restrict + the set of sstables we check (CASSANDRA-4116) + * incremental repair by token range (CASSANDRA-3912) + * streaming commitlog backup + pitr (CASSANDRA-3690) + * avoid generating redundant compaction tasks during streaming + (CASSANDRA-4174) + * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to + StorageService mbean (CASSANDRA-556) + * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079) + * optimize truncate when autosnapshot is disabled (CASSANDRA-4153) + * add support for commitlog archiving and point-in-time recovery + (CASSANDRA-3647) + * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966) + * add column limit to cli (CASSANDRA-3012, 4098) + * clean up and optimize DataOutputBuffer, used by CQL compression and + CompositeType (CASSANDRA-4072) + * optimize commitlog checksumming (CASSANDRA-3610) + * identify and blacklist corrupted SSTables from future compactions + (CASSANDRA-2261) + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037) + * Expose repairing by a user provided range (CASSANDRA-3912) + * Add way to force the cassandra-cli to refresh it's schema (CASSANDRA-4052) + * Avoids having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889) + * (cql3) Fix order by for reversed queries (CASSANDRA-4160) + * (cql3) Add ReversedType support (CASSANDRA-4004) + * (cql3) Add timeuuid type (CASSANDRA-4194) + * (cql3) Minor fixes (CASSANDRA-4185) + * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202) + * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186) + * (cql3) Move max/min compaction thresholds to compaction strategy options + (CASSANDRA-4187) +Merged from 1.0: + * Fix super columns bug where cache is not updated (CASSANDRA-4190) * fix maxTimestamp to include row tombstones (CASSANDRA-4116) + * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129) + * Avoids possible deadlock during bootstrap (CASSANDRA-4159) + * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128) + * Fix super columns bug where cache is not updated (CASSANDRA-4190) + * stress tool to return appropriate exit code on failure (CASSANDRA-4188) ++ * fix compaction NPE when out of disk space and assertions disabled ++ (CASSANDRA-3985) + + +1.1.0-final + * average a reduced liveRatio estimate with the previous one (CASSANDRA-4065) + * Allow KS and CF names up to 48 characters (CASSANDRA-4157) + * fix stress build (CASSANDRA-4140) + * add time remaining estimate to nodetool compactionstats (CASSANDRA-4167) + * (cql) fix NPE in cql3 ALTER TABLE (CASSANDRA-4163) + * (cql) Add support for CL.TWO and CL.THREE in CQL (CASSANDRA-4156) + * (cql) Fix type in CQL3 ALTER TABLE preventing update (CASSANDRA-4170) + * (cql) Throw invalid exception from CQL3 on obsolete options (CASSANDRA-4171) + * (cqlsh) fix recognizing uppercase SELECT keyword (CASSANDRA-4161) + * Pig: wide row support (CASSANDRA-3909) +Merged from 1.0: * avoid streaming empty files with bulk loader if sstablewriter errors out (CASSANDRA-3946) + + +1.1-rc1 + * Include stress tool in binary builds (CASSANDRA-4103) + * (Hadoop) fix wide row iteration when last row read was deleted + (CASSANDRA-4154) + * fix read_repair_chance to really default to 0.1 in the cli (CASSANDRA-4114) + * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042) + * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087) + * fix KEYS index from skipping results (CASSANDRA-3996) + * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076) + * make loadNewSStable preserve sstable version (CASSANDRA-4077) + * Respect 1.0 cache settings as much as possible when upgrading + (CASSANDRA-4088) + * relax path length requirement for sstable files when upgrading on + non-Windows platforms (CASSANDRA-4110) + * fix terminination of the stress.java when errors were encountered + (CASSANDRA-4128) + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037) + * Fix get_paged_slice (CASSANDRA-4136) + * CQL3: Support slice with exclusive start and stop (CASSANDRA-3785) +Merged from 1.0: * support PropertyFileSnitch in bulk loader (CASSANDRA-4145) * add auto_snapshot option allowing disabling snapshot before drop/truncate (CASSANDRA-3710) * allow short snitch names (CASSANDRA-4130) + + +1.1-beta2 + * rename loaded sstables to avoid conflicts with local snapshots + (CASSANDRA-3967) + * start hint replay as soon as FD notifies that the target is back up + (CASSANDRA-3958) + * avoid unproductive deserializing of cached rows during compaction + (CASSANDRA-3921) + * fix concurrency issues with CQL keyspace creation (CASSANDRA-3903) + * Show Effective Owership via Nodetool ring <keyspace> (CASSANDRA-3412) + * Update ORDER BY syntax for CQL3 (CASSANDRA-3925) + * Fix BulkRecordWriter to not throw NPE if reducer gets no map data from Hadoop (CASSANDRA-3944) + * Fix bug with counters in super columns (CASSANDRA-3821) + * Remove deprecated merge_shard_chance (CASSANDRA-3940) + * add a convenient way to reset a node's schema (CASSANDRA-2963) + * fix for intermittent SchemaDisagreementException (CASSANDRA-3884) + * CLI `list <CF>` to limit number of columns and their order (CASSANDRA-3012) + * ignore deprecated KsDef/CfDef/ColumnDef fields in native schema (CASSANDRA-3963) + * CLI to report when unsupported column_metadata pair was given (CASSANDRA-3959) + * reincarnate removed and deprecated KsDef/CfDef attributes (CASSANDRA-3953) + * Fix race between writes and read for cache (CASSANDRA-3862) + * perform static initialization of StorageProxy on start-up (CASSANDRA-3797) + * support trickling fsync() on writes (CASSANDRA-3950) + * expose counters for unavailable/timeout exceptions given to thrift clients (CASSANDRA-3671) + * avoid quadratic startup time in LeveledManifest (CASSANDRA-3952) + * Add type information to new schema_ columnfamilies and remove thrift + serialization for schema (CASSANDRA-3792) + * add missing column validator options to the CLI help (CASSANDRA-3926) + * skip reading saved key cache if CF's caching strategy is NONE or ROWS_ONLY (CASSANDRA-3954) + * Unify migration code (CASSANDRA-4017) +Merged from 1.0: * cqlsh: guess correct version of Python for Arch Linux (CASSANDRA-4090) + * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129) + * Avoids possible deadlock during bootstrap (CASSANDRA-4159) + * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128) + * Fix super columns bug where cache is not updated (CASSANDRA-4190) + * stress tool to return appropriate exit code on failure (CASSANDRA-4188) - * fix compaction NPE when out of disk space and assertions disabled - (CASSANDRA-3985) 1.0.9 http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/Directories.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/Directories.java index 6d16559,0000000..f023c8c mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@@ -1,584 -1,0 +1,571 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOError; +import java.io.IOException; +import java.util.*; + +import org.apache.commons.lang.StringUtils; +import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.*; +import org.apache.cassandra.db.compaction.LeveledManifest; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.MmappedSegmentedFile; +import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.CLibrary; +import org.apache.cassandra.utils.Pair; + +/** + * Encapsulate handling of paths to the data files. + * + * The directory layout is the following: + * /<path_to_data_dir>/ks/cf1/ks-cf1-hb-1-Data.db + * /cf2/ks-cf2-hb-1-Data.db + * ... + * + * In addition, more that one 'root' data directory can be specified so that + * <path_to_data_dir> potentially represents multiple locations. + * Note that in the case of multiple locations, the manifest for the leveled + * compaction is only in one of the location. + * + * Snapshots (resp. backups) are always created along the sstables thare are + * snapshoted (resp. backuped) but inside a subdirectory named 'snapshots' + * (resp. backups) (and snapshots are furter inside a subdirectory of the name + * of the snapshot). + * + * This class abstracts all those details from the rest of the code. + */ +public class Directories +{ + private static Logger logger = LoggerFactory.getLogger(Directories.class); + + public static final String BACKUPS_SUBDIR = "backups"; + public static final String SNAPSHOT_SUBDIR = "snapshots"; + public static final char SECONDARY_INDEX_NAME_SEPARATOR = '.'; + + public static final File[] dataFileLocations; + static + { + String[] locations = DatabaseDescriptor.getAllDataFileLocations(); + dataFileLocations = new File[locations.length]; + for (int i = 0; i < locations.length; ++i) + dataFileLocations[i] = new File(locations[i]); + } + + private final String tablename; + private final String cfname; + private final File[] sstableDirectories; + + public static Directories create(String tablename, String cfname) + { + int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); + if (idx > 0) + // secondary index, goes in the same directory than the base cf + return new Directories(tablename, cfname, cfname.substring(0, idx)); + else + return new Directories(tablename, cfname, cfname); + } + + private Directories(String tablename, String cfname, String directoryName) + { + this.tablename = tablename; + this.cfname = cfname; + this.sstableDirectories = new File[dataFileLocations.length]; + for (int i = 0; i < dataFileLocations.length; ++i) + sstableDirectories[i] = new File(dataFileLocations[i], join(tablename, directoryName)); + + if (!StorageService.instance.isClientMode()) + { + try + { + for (File dir : sstableDirectories) + FileUtils.createDirectory(dir); + } + catch (IOException e) + { + throw new IOError(e); + } + } + } + + public File getDirectoryForNewSSTables(long estimatedSize) + { - return getDirectoryForNewSSTables(estimatedSize, true); - } ++ File path = getLocationWithMaximumAvailableSpace(estimatedSize); + - public File getDirectoryForNewSSTables(long estimatedSize, boolean ensureFreeSpace) - { - File path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace); + // Requesting GC has a chance to free space only if we're using mmap and a non SUN jvm + if (path == null - && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap || DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap) - && !MmappedSegmentedFile.isCleanerAvailable()) ++ && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap || DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap) ++ && !MmappedSegmentedFile.isCleanerAvailable()) + { ++ logger.info("Forcing GC to free up disk space. Upgrade to the Oracle JVM to avoid this"); + StorageService.instance.requestGC(); + // retry after GCing has forced unmap of compacted SSTables so they can be deleted + // Note: GCInspector will do this already, but only sun JVM supports GCInspector so far + SSTableDeletingTask.rescheduleFailedTasks(); + try + { + Thread.sleep(10000); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } - path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace); ++ path = getLocationWithMaximumAvailableSpace(estimatedSize); + } ++ + return path; + } + + /* + * Loop through all the disks to see which disk has the max free space + * return the disk with max free space for compactions. If the size of the expected + * compacted file is greater than the max disk space available return null, we cannot + * do compaction in this case. + */ - public File getLocationWithMaximumAvailableSpace(long estimatedSize, boolean ensureFreeSpace) ++ public File getLocationWithMaximumAvailableSpace(long estimatedSize) + { + long maxFreeDisk = 0; + File maxLocation = null; + + for (File dir : sstableDirectories) + { + if (maxFreeDisk < dir.getUsableSpace()) + { + maxFreeDisk = dir.getUsableSpace(); + maxLocation = dir; + } + } - logger.debug(String.format("expected data files size is %d; largest free partition (%s) has %d bytes free", - estimatedSize, - maxLocation, - maxFreeDisk)); - + // Load factor of 0.9 we do not want to use the entire disk that is too risky. + maxFreeDisk = (long) (0.9 * maxFreeDisk); ++ logger.debug(String.format("expected data files size is %d; largest free partition (%s) has %d bytes free", ++ estimatedSize, maxLocation, maxFreeDisk)); + - if (!ensureFreeSpace || estimatedSize < maxFreeDisk) - { - if (estimatedSize >= maxFreeDisk) - logger.warn(String.format("Data file location %s only has %d free, estimated size is %d", - maxLocation, - maxFreeDisk, - estimatedSize)); + ++ if (estimatedSize < maxFreeDisk) + return maxLocation; - } - + return null; + } + + public static File getSnapshotDirectory(Descriptor desc, String snapshotName) + { + return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName); + } + + public static File getBackupsDirectory(Descriptor desc) + { + return getOrCreate(desc.directory, BACKUPS_SUBDIR); + } + + public SSTableLister sstableLister() + { + return new SSTableLister(); + } + + public class SSTableLister + { + private boolean skipCompacted; + private boolean skipTemporary; + private boolean includeBackups; + private boolean onlyBackups; + private int nbFiles; + private final Map<Descriptor, Set<Component>> components = new HashMap<Descriptor, Set<Component>>(); + private boolean filtered; + private String snapshotName; + + public SSTableLister skipCompacted(boolean b) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + skipCompacted = b; + return this; + } + + public SSTableLister skipTemporary(boolean b) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + skipTemporary = b; + return this; + } + + public SSTableLister includeBackups(boolean b) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + includeBackups = b; + return this; + } + + public SSTableLister onlyBackups(boolean b) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + onlyBackups = b; + includeBackups = b; + return this; + } + + public SSTableLister snapshots(String sn) + { + if (filtered) + throw new IllegalStateException("list() has already been called"); + snapshotName = sn; + return this; + } + + public Map<Descriptor, Set<Component>> list() + { + filter(); + return ImmutableMap.copyOf(components); + } + + public List<File> listFiles() + { + filter(); + List<File> l = new ArrayList<File>(nbFiles); + for (Map.Entry<Descriptor, Set<Component>> entry : components.entrySet()) + { + for (Component c : entry.getValue()) + { + l.add(new File(entry.getKey().filenameFor(c))); + } + } + return l; + } + + private void filter() + { + if (filtered) + return; + + for (File location : sstableDirectories) + { + if (snapshotName != null) + { + new File(location, join(SNAPSHOT_SUBDIR, snapshotName)).listFiles(getFilter()); + continue; + } + + if (!onlyBackups) + location.listFiles(getFilter()); + + if (includeBackups) + new File(location, BACKUPS_SUBDIR).listFiles(getFilter()); + } + filtered = true; + } + + private FileFilter getFilter() + { + // Note: the prefix needs to include cfname + separator to distinguish between a cfs and it's secondary indexes + final String sstablePrefix = tablename + Component.separator + cfname + Component.separator; + return new FileFilter() + { + // This function always return false since accepts adds to the components map + public boolean accept(File file) + { + // we are only interested in the SSTable files that belong to the specific ColumnFamily + if (file.isDirectory() || !file.getName().startsWith(sstablePrefix)) + return false; + + Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName()); + if (pair == null) + return false; + + if (skipCompacted && new File(pair.left.filenameFor(Component.COMPACTED_MARKER)).exists()) + return false; + if (skipTemporary && pair.left.temporary) + return false; + + Set<Component> previous = components.get(pair.left); + if (previous == null) + { + previous = new HashSet<Component>(); + components.put(pair.left, previous); + } + previous.add(pair.right); + nbFiles++; + return false; + } + }; + } + } + + public File tryGetLeveledManifest() + { + for (File dir : sstableDirectories) + { + File manifestFile = new File(dir, cfname + LeveledManifest.EXTENSION); + if (manifestFile.exists()) + { + logger.debug("Found manifest at {}", manifestFile); + return manifestFile; + } + } + logger.debug("No level manifest found"); + return null; + } + + public File getOrCreateLeveledManifest() + { + File manifestFile = tryGetLeveledManifest(); + if (manifestFile == null) + manifestFile = new File(sstableDirectories[0], cfname + LeveledManifest.EXTENSION); + return manifestFile; + } + + public void snapshotLeveledManifest(String snapshotName) throws IOException + { + File manifest = tryGetLeveledManifest(); + if (manifest != null) + { + File snapshotDirectory = getOrCreate(manifest.getParentFile(), SNAPSHOT_SUBDIR, snapshotName); + CLibrary.createHardLink(manifest, new File(snapshotDirectory, manifest.getName())); + } + } + + public boolean snapshotExists(String snapshotName) + { + for (File dir : sstableDirectories) + { + File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName)); + if (snapshotDir.exists()) + return true; + } + return false; + } + + public void clearSnapshot(String snapshotName) throws IOException + { + // If snapshotName is empty or null, we will delete the entire snapshot directory + String tag = snapshotName == null ? "" : snapshotName; + for (File dir : sstableDirectories) + { + File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag)); + if (snapshotDir.exists()) + { + if (logger.isDebugEnabled()) + logger.debug("Removing snapshot directory " + snapshotDir); + FileUtils.deleteRecursive(snapshotDir); + } + } + } + + private static File getOrCreate(File base, String... subdirs) + { + File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs)); + if (dir.exists()) + { + if (!dir.isDirectory()) + throw new IOError(new IOException(String.format("Invalid directory path %s: path exists but is not a directory", dir))); + } + else if (!dir.mkdirs()) + { + throw new IOError(new IOException("Unable to create directory " + dir)); + } + return dir; + } + + private static String join(String... s) + { + return StringUtils.join(s, File.separator); + } + + /** + * To check if sstables needs migration, we look at the System directory. + * If it contains a directory for the status cf, we'll attempt a sstable + * migration. + * Note that it is mostly harmless to try a migration uselessly, except + * maybe for some wasted cpu cycles. + */ + public static boolean sstablesNeedsMigration() + { + if (StorageService.instance.isClientMode()) + return false; + + boolean hasSystemKeyspace = false; + for (File location : dataFileLocations) + { + File systemDir = new File(location, Table.SYSTEM_TABLE); + hasSystemKeyspace |= (systemDir.exists() && systemDir.isDirectory()); + File statusCFDir = new File(systemDir, SystemTable.STATUS_CF); + if (statusCFDir.exists()) + return false; + } + if (!hasSystemKeyspace) + // This is a brand new node. + return false; + + // Check whether the migration might create too long a filename + int longestLocation = -1; + try + { + for (File loc : dataFileLocations) + longestLocation = Math.max(longestLocation, loc.getCanonicalPath().length()); + } + catch (IOException e) + { + throw new IOError(e); + } + + // Check that migration won't error out halfway through from too-long paths. For Windows, we need to check + // total path length <= 255 (see http://msdn.microsoft.com/en-us/library/aa365247.aspx and discussion on CASSANDRA-2749); + // elsewhere, we just need to make sure filename is <= 255. + for (KSMetaData ksm : Schema.instance.getTableDefinitions()) + { + String ksname = ksm.name; + for (Map.Entry<String, CFMetaData> entry : ksm.cfMetaData().entrySet()) + { + String cfname = entry.getKey(); + + // max path is roughly (guess-estimate) <location>/ksname/cfname/snapshots/1324314347102-somename/ksname-cfname-tmp-hb-65536-Statistics.db + if (System.getProperty("os.name").startsWith("Windows") + && longestLocation + (ksname.length() + cfname.length()) * 2 + 63 > 255) + { + throw new RuntimeException(String.format("Starting with 1.1, keyspace names and column family " + + "names must be less than %s characters long. %s/%s doesn't" + + " respect that restriction. Please rename your " + + "keyspace/column families to respect that restriction " + + "before updating.", Schema.NAME_LENGTH, ksname, cfname)); + } + + if (ksm.name.length() + cfname.length() + 28 > 255) + { + throw new RuntimeException("Starting with 1.1, the keyspace name is included in data filenames. For " + + ksm.name + "/" + cfname + ", this puts you over the largest possible filename of 255 characters"); + } + } + } + + return true; + } + + /** + * Move sstables from the pre-#2749 layout to their new location/names. + * This involves: + * - moving each sstable to their CF specific directory + * - rename the sstable to include the keyspace in the filename + * + * Note that this also move leveled manifests, snapshots and backups. + */ + public static void migrateSSTables() + { + logger.info("Upgrade from pre-1.1 version detected: migrating sstables to new directory layout"); + + for (File location : dataFileLocations) + { + if (!location.exists() || !location.isDirectory()) + continue; + + for (File ksDir : location.listFiles()) + { + if (!ksDir.isDirectory()) + continue; + + for (File file : ksDir.listFiles()) + migrateFile(file, ksDir, null); + + migrateSnapshots(ksDir); + migrateBackups(ksDir); + } + } + } + + private static void migrateSnapshots(File ksDir) + { + File snapshotDir = new File(ksDir, SNAPSHOT_SUBDIR); + if (!snapshotDir.exists()) + return; + + for (File snapshot : snapshotDir.listFiles()) + { + if (!snapshot.isDirectory()) + continue; + + for (File f : snapshot.listFiles()) + migrateFile(f, ksDir, join(SNAPSHOT_SUBDIR, snapshot.getName())); + + if (!snapshot.delete()) + logger.info("Old snapsot directory {} not deleted by migraation as it is not empty", snapshot); + } + if (!snapshotDir.delete()) + logger.info("Old directory {} not deleted by migration as it is not empty", snapshotDir); + } + + private static void migrateBackups(File ksDir) + { + File backupDir = new File(ksDir, BACKUPS_SUBDIR); + if (!backupDir.exists()) + return; + + for (File f : backupDir.listFiles()) + migrateFile(f, ksDir, BACKUPS_SUBDIR); + + if (!backupDir.delete()) + logger.info("Old directory {} not deleted by migration as it is not empty", backupDir); + } + + private static void migrateFile(File file, File ksDir, String additionalPath) + { + try + { + if (file.isDirectory()) + return; + + String name = file.getName(); + boolean isManifest = name.endsWith(LeveledManifest.EXTENSION); + String cfname = isManifest + ? name.substring(0, name.length() - LeveledManifest.EXTENSION.length()) + : name.substring(0, name.indexOf(Component.separator)); + + int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); // idx > 0 => secondary index + String dirname = idx > 0 ? cfname.substring(0, idx) : cfname; + File destDir = getOrCreate(ksDir, dirname, additionalPath); + + File destFile = new File(destDir, isManifest ? name : ksDir.getName() + Component.separator + name); + logger.debug(String.format("[upgrade to 1.1] Moving %s to %s", file, destFile)); + FileUtils.renameWithConfirm(file, destFile); + } + catch (IOException e) + { + throw new IOError(e); + } + } + + // Hack for tests, don't use otherwise + static void overrideDataDirectoriesForTest(String loc) + { + for (int i = 0; i < dataFileLocations.length; ++i) + dataFileLocations[i] = new File(loc); + } + + // Hack for tests, don't use otherwise + static void resetDataDirectoriesAfterTest() + { + String[] locations = DatabaseDescriptor.getAllDataFileLocations(); + for (int i = 0; i < locations.length; ++i) + dataFileLocations[i] = new File(locations[i]); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java index b66f20b,2a1b415..6d914e3 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@@ -18,10 -18,9 +18,9 @@@ package org.apache.cassandra.db.compaction; +import java.io.File; import java.io.IOException; import java.util.*; --import java.util.Map.Entry; import com.google.common.base.Predicates; import com.google.common.collect.Iterators; @@@ -75,9 -76,9 +74,7 @@@ public class CompactionTask extends Abs if (!isCompactionInteresting(toCompact)) return 0; - File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact), - ensureFreeSpace()); - - if (compactionFileLocation == null) - compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact)); - ++ File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact)); if (compactionFileLocation == null && partialCompactionsAcceptable()) { // If the compaction file path is null that means we have no space left for this compaction. @@@ -88,17 -89,14 +85,15 @@@ // Note that we have removed files that are still marked as compacting. // This suboptimal but ok since the caller will unmark all the sstables at the end. toCompact.remove(cfs.getMaxSizeFile(toCompact)); - compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact), - ensureFreeSpace()); - compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact)); ++ compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact)); } } + if (compactionFileLocation == null) { - logger.warn("insufficient space to compact even the two smallest files, aborting"); + logger.warn("insufficient space to compact; aborting compaction"); return 0; } - assert compactionFileLocation != null; if (DatabaseDescriptor.isSnapshotBeforeCompaction()) cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-" + "compact-" + cfs.columnFamily); @@@ -204,10 -198,10 +199,10 @@@ cfs.replaceCompactedSSTables(toCompact, sstables, compactionType); // TODO: this doesn't belong here, it should be part of the reader to load when the tracker is wired up -- for (Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet()) ++ for (Map.Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet()) { SSTableReader key = ssTableReaderMapEntry.getKey(); -- for (Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet()) ++ for (Map.Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet()) key.cacheKey(entry.getKey(), entry.getValue()); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/Descriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java ----------------------------------------------------------------------
