Author: ecn
Date: Tue Jun 18 22:14:06 2013
New Revision: 1494340
URL: http://svn.apache.org/r1494340
Log:
ACCUMULO-118 cleanup TODOs simplify VolumeManager interface, add some basic
documentation
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
Tue Jun 18 22:14:06 2013
@@ -28,7 +28,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
- * A wrapper around multiple hadoop FileSystem objects, which are assumed to
be different namespaces.
+ * A wrapper around multiple hadoop FileSystem objects, which are assumed to
be different volumes.
+ * This also concentrates a bunch of meta-operations like waiting for
SAFE_MODE, and closing WALs.
*/
public interface VolumeManager {
@@ -68,12 +69,10 @@ public interface VolumeManager {
// find the appropriate FileSystem object given a path
FileSystem getFileSystemByPath(Path path);
- FileSystem getFileSystemByPath(String path);
-
// get a mapping of volume to FileSystem
Map<String, ? extends FileSystem> getFileSystems();
- // return the item in options that is in the same namespace as source
+ // return the item in options that is in the same volume as source
Path matchingFileSystem(Path source, String[] options);
// create a new path in the same volume as the sourceDir
@@ -88,7 +87,7 @@ public interface VolumeManager {
// forward to the appropriate FileSystem object
FSDataInputStream open(Path path) throws IOException;
- // forward to the appropriate FileSystem object, throws an exception if the
paths are in different namespaces
+ // forward to the appropriate FileSystem object, throws an exception if the
paths are in different volumes
boolean rename(Path path, Path newPath) throws IOException;
// forward to the appropriate FileSystem object
@@ -118,5 +117,6 @@ public interface VolumeManager {
// forward to the appropriate FileSystem object
ContentSummary getContentSummary(Path dir) throws IOException;
+ // decide on which of the given locations to create a new file
String choose(String[] options);
}
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
Tue Jun 18 22:14:06 2013
@@ -260,11 +260,6 @@ public class VolumeManagerImpl implement
}
@Override
- public FileSystem getFileSystemByPath(String path) {
- return getFileSystemByPath(new Path(path));
- }
-
- @Override
public Map<String, ? extends FileSystem> getFileSystems() {
return volumes;
}
@@ -399,9 +394,9 @@ public class VolumeManagerImpl implement
relPath = relPath.substring(2);
else
relPath = "/" + new String(tableId) + relPath;
- String fullPath = ServerConstants.getTablesDirs()[0] + relPath;
+ Path fullPath = new Path(ServerConstants.getTablesDirs()[0] + relPath);
FileSystem fs = getFileSystemByPath(fullPath);
- return fs.makeQualified(new Path(fullPath));
+ return fs.makeQualified(fullPath);
}
@Override
@@ -440,8 +435,8 @@ public class VolumeManagerImpl implement
fullPath = path + fileName;
else
fullPath = path + "/" + fileName;
- FileSystem ns = getFileSystemByPath(fullPath);
Path exists = new Path(fullPath);
+ FileSystem ns = getFileSystemByPath(exists);
if (ns.exists(exists)) {
Path result = ns.makeQualified(exists);
log.debug("Found " + exists + " on " + path + " as " + result);
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
Tue Jun 18 22:14:06 2013
@@ -128,7 +128,6 @@ public class RecoveryManager {
String filename = hostFilename[1];
String parts[] = filename.split("/");
String sortId = parts[parts.length - 1];
- // TODO: ACCUMULO-118: choose recovery directory with extension
String dest =
master.getFileSystem().choose(ServerConstants.getRecoveryDirs()) + "/" + sortId;
log.debug("Recovering " + filename + " to " + dest);
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
Tue Jun 18 22:14:06 2013
@@ -284,7 +284,7 @@ public class Compactor implements Callab
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns =
this.fs.getFileSystemByPath(outputFile.path().toString());
+ FileSystem ns = this.fs.getFileSystemByPath(outputFile.path());
mfw = fileFactory.openWriter(outputFile.path().toString(), ns,
ns.getConf(), acuTableConf);
Map<String,Set<ByteSequence>> lGroups;
@@ -365,7 +365,7 @@ public class Compactor implements Callab
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem fs = this.fs.getFileSystemByPath(mapFile.path().toString());
+ FileSystem fs = this.fs.getFileSystemByPath(mapFile.path());
FileSKVIterator reader;
reader = fileFactory.openReader(mapFile.path().toString(), false, fs,
conf, acuTableConf);
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Tue Jun 18 22:14:06 2013
@@ -650,10 +650,17 @@ public class Tablet {
for (FileRef tpath : paths.keySet()) {
- // TODO ACCUMULO-118
-// if (!tpath.getParent().getParent().equals(new
Path(ServerConstants.getTablesDir() + "/" + extent.getTableId()))) {
-// throw new IOException("Map file " + tpath + " not in table dir " +
ServerConstants.getTablesDir() + "/" + extent.getTableId());
-// }
+ boolean inTheRightDirectory = false;
+ Path parent = tpath.path().getParent().getParent();
+ for (String tablesDir : ServerConstants.getTablesDirs()) {
+ if (parent.equals(new Path(tablesDir,
extent.getTableId().toString()))) {
+ inTheRightDirectory = true;
+ break;
+ }
+ }
+ if (!inTheRightDirectory) {
+ throw new IOException("Map file " + tpath + " not in table dirs");
+ }
if (bulkDir == null)
bulkDir = tpath.path().getParent().toString();
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Tue Jun 18 22:14:06 2013
@@ -882,8 +882,9 @@ public class TabletServer extends Abstra
Map<String,MapFileInfo> fileMap = entry.getValue();
Map<FileRef, MapFileInfo> fileRefMap = new HashMap<FileRef,
MapFileInfo>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
- FileSystem ns = fs.getFileSystemByPath(mapping.getKey());
- Path path = ns.makeQualified(new Path(mapping.getKey()));
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getFileSystemByPath(path);
+ path = ns.makeQualified(path);
fileRefMap.put(new FileRef(path.toString(), path),
mapping.getValue());
}
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Tue Jun 18 22:14:06 2013
@@ -196,9 +196,9 @@ public class LogSorter {
}
private void writeBuffer(String destPath,
ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
- String path = destPath + String.format("/part-r-%05d", part++);
+ Path path = new Path(destPath, String.format("part-r-%05d", part++));
FileSystem ns = fs.getFileSystemByPath(path);
- MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path,
LogFileKey.class, LogFileValue.class);
+ MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns,
path.toString(), LogFileKey.class, LogFileValue.class);
try {
Collections.sort(buffer, new
Comparator<Pair<LogFileKey,LogFileValue>>() {
@Override
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
Tue Jun 18 22:14:06 2013
@@ -121,11 +121,11 @@ public class FileUtil {
start = end;
- String newMapFile = String.format("%s/%04d." + RFile.EXTENSION, newDir,
count++);
+ FileRef newMapFile = new FileRef(String.format("%s/%04d." +
RFile.EXTENSION, newDir, count++));
- outFiles.add(new FileRef(newMapFile));
- FileSystem ns = fs.getFileSystemByPath(newMapFile);
- FileSKVWriter writer = new RFileOperations().openWriter(newMapFile, ns,
ns.getConf(), acuConf);
+ outFiles.add(newMapFile);
+ FileSystem ns = fs.getFileSystemByPath(newMapFile.path());
+ FileSKVWriter writer = new
RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf);
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new
ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
@@ -434,10 +434,10 @@ public class FileUtil {
for (FileRef mapfile : mapfiles) {
FileSKVIterator reader = null;
- String path = mapfile.path().toString();
- FileSystem ns = fs.getFileSystemByPath(path);
+ //String path = mapfile.path().toString();
+ FileSystem ns = fs.getFileSystemByPath(mapfile.path());
try {
- reader = FileOperations.getInstance().openReader(path, false, ns,
ns.getConf(), acuConf);
+ reader = FileOperations.getInstance().openReader(mapfile.toString(),
false, ns, ns.getConf(), acuConf);
Key firstKey = reader.getFirstKey();
if (firstKey != null) {
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
Tue Jun 18 22:14:06 2013
@@ -68,7 +68,7 @@ public class OfflineMetadataScanner exte
private List<SortedKeyValueIterator<Key,Value>>
openMapFiles(Collection<String> files, VolumeManager fs, AccumuloConfiguration
conf) throws IOException {
List<SortedKeyValueIterator<Key,Value>> readers = new
ArrayList<SortedKeyValueIterator<Key,Value>>();
for (String file : files) {
- FileSystem ns = fs.getFileSystemByPath(file);
+ FileSystem ns = fs.getFileSystemByPath(new Path(file));
FileSKVIterator reader = FileOperations.getInstance().openReader(file,
true, ns, ns.getConf(), conf);
readers.add(reader);
}
Modified:
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
Tue Jun 18 22:14:06 2013
@@ -123,7 +123,7 @@ public class SortedLogRecoveryTest {
try {
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getFileSystemByPath(new Path(path));
Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1",
LogFileKey.class, LogFileValue.class);
for (KeyValue lfe : entry.getValue()) {
map.append(lfe.key, lfe.value);
