Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java Tue Jun 18 17:45:01 2013 @@ -45,8 +45,9 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.server.fs.FileRef; -import org.apache.accumulo.server.fs.FileSystem; +import org.apache.accumulo.server.fs.VolumeManager; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; @@ -74,7 +75,7 @@ public class FileUtil { private static final Logger log = Logger.getLogger(FileUtil.class); - private static String createTmpDir(AccumuloConfiguration acuConf, FileSystem fs) throws IOException { + private static String createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException { String accumuloDir = acuConf.get(Property.INSTANCE_DFS_DIR); String tmpDir = null; @@ -99,7 +100,7 @@ public class FileUtil { return tmpDir; } - public static Collection<FileRef> reduceFiles(AccumuloConfiguration acuConf, Configuration conf, FileSystem fs, Text prevEndRow, Text endRow, + public static Collection<FileRef> reduceFiles(AccumuloConfiguration acuConf, Configuration conf, VolumeManager fs, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, int maxFiles, String tmpDir, int pass) throws IOException { ArrayList<FileRef> paths = new ArrayList<FileRef>(mapFiles); @@ -123,7 +124,7 @@ public class FileUtil { String newMapFile = String.format("%s/%04d." + RFile.EXTENSION, newDir, count++); outFiles.add(new FileRef(newMapFile)); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(newMapFile); + FileSystem ns = fs.getFileSystemByPath(newMapFile); FileSKVWriter writer = new RFileOperations().openWriter(newMapFile, ns, ns.getConf(), acuConf); writer.startDefaultLocalityGroup(); List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size()); @@ -181,12 +182,12 @@ public class FileUtil { return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1); } - public static SortedMap<Double,Key> findMidPoint(FileSystem fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, + public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, double minSplit) throws IOException { return findMidPoint(fs, acuConf, prevEndRow, endRow, mapFiles, minSplit, true); } - public static double estimatePercentageLTE(FileSystem fs, AccumuloConfiguration acuconf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, + public static double estimatePercentageLTE(VolumeManager fs, AccumuloConfiguration acuconf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, Text splitRow) throws IOException { Configuration conf = CachedConfiguration.getInstance(); @@ -258,7 +259,7 @@ public class FileUtil { * ISSUES : This method used the index files to find the mid point. If the map files have different index intervals this method will not return an * accurate mid point. Also, it would be tricky to use this method in conjunction with an in memory map because the indexing interval is unknown. */ - public static SortedMap<Double,Key> findMidPoint(FileSystem fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, + public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, double minSplit, boolean useIndex) throws IOException { Configuration conf = CachedConfiguration.getInstance(); @@ -358,7 +359,7 @@ public class FileUtil { } } - private static void cleanupIndexOp(AccumuloConfiguration acuConf, String tmpDir, FileSystem fs, ArrayList<FileSKVIterator> readers) throws IOException { + private static void cleanupIndexOp(AccumuloConfiguration acuConf, String tmpDir, VolumeManager fs, ArrayList<FileSKVIterator> readers) throws IOException { // close all of the index sequence files for (FileSKVIterator r : readers) { try { @@ -380,7 +381,7 @@ public class FileUtil { } private static long countIndexEntries(AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<FileRef> mapFiles, boolean useIndex, - Configuration conf, FileSystem fs, ArrayList<FileSKVIterator> readers) throws IOException { + Configuration conf, VolumeManager fs, ArrayList<FileSKVIterator> readers) throws IOException { long numKeys = 0; @@ -388,7 +389,7 @@ public class FileUtil { for (FileRef ref : mapFiles) { FileSKVIterator reader = null; Path path = ref.path(); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getFileSystemByPath(path); try { if (useIndex) reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf); @@ -424,7 +425,7 @@ public class FileUtil { return numKeys; } - public static Map<FileRef,FileInfo> tryToGetFirstAndLastRows(FileSystem fs, AccumuloConfiguration acuConf, Set<FileRef> mapfiles) { + public static Map<FileRef,FileInfo> tryToGetFirstAndLastRows(VolumeManager fs, AccumuloConfiguration acuConf, Set<FileRef> mapfiles) { HashMap<FileRef,FileInfo> mapFilesInfo = new HashMap<FileRef,FileInfo>(); @@ -434,7 +435,7 @@ public class FileUtil { FileSKVIterator reader = null; String path = mapfile.path().toString(); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getFileSystemByPath(path); try { reader = FileOperations.getInstance().openReader(path, false, ns, ns.getConf(), acuConf); @@ -464,12 +465,12 @@ public class FileUtil { return mapFilesInfo; } - public static WritableComparable<Key> findLastKey(FileSystem fs, AccumuloConfiguration acuConf, Collection<FileRef> mapFiles) throws IOException { + public static WritableComparable<Key> findLastKey(VolumeManager fs, AccumuloConfiguration acuConf, Collection<FileRef> mapFiles) throws IOException { Key lastKey = null; for (FileRef ref : mapFiles) { Path path = ref.path(); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getFileSystemByPath(path); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf); try { @@ -504,7 +505,7 @@ public class FileUtil { } public static Map<KeyExtent,Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize, List<KeyExtent> extents, Configuration conf, - FileSystem fs) throws IOException { + VolumeManager fs) throws IOException { long totalIndexEntries = 0; Map<KeyExtent,MLong> counts = new TreeMap<KeyExtent,MLong>(); @@ -512,7 +513,7 @@ public class FileUtil { counts.put(keyExtent, new MLong(0)); Text row = new Text(); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(mapFile); + FileSystem ns = fs.getFileSystemByPath(mapFile); FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf); try {
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/Initialize.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/Initialize.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/Initialize.java Tue Jun 18 17:45:01 2013 @@ -53,8 +53,8 @@ import org.apache.accumulo.server.Server import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.constraints.MetadataConstraints; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter; import org.apache.accumulo.server.master.state.tables.TableManager; import org.apache.accumulo.server.security.AuditedSecurityOperation; @@ -63,6 +63,7 @@ import org.apache.accumulo.server.tablet import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; @@ -114,11 +115,11 @@ public class Initialize { initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true"); } - public static boolean doInit(Opts opts, Configuration conf, FileSystem fs) throws IOException { + public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException { if (!ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI).equals("")) log.info("Hadoop Filesystem is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI)); else - log.info("Hadoop Filesystem is " + org.apache.hadoop.fs.FileSystem.getDefaultUri(conf)); + log.info("Hadoop Filesystem is " + FileSystem.getDefaultUri(conf)); log.info("Accumulo data dirs are " + Arrays.asList(ServerConstants.getBaseDirs())); log.info("Zookeeper server is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST)); @@ -162,7 +163,7 @@ public class Initialize { return initialize(opts, instanceNamePath, fs); } - public static boolean initialize(Opts opts, String instanceNamePath, FileSystem fs) { + public static boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs) { UUID uuid = UUID.randomUUID(); try { @@ -220,7 +221,7 @@ public class Initialize { return result.toArray(a); } - private static void initFileSystem(Opts opts, FileSystem fs, UUID uuid) throws IOException { + private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid) throws IOException { FileStatus fstat; // the actual disk location of the root tablet @@ -275,7 +276,7 @@ public class Initialize { // metadata tablets String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration()); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile)); + FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile)); FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration()); mfw.startDefaultLocalityGroup(); @@ -477,7 +478,7 @@ public class Initialize { initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), rep); } - public static boolean isInitialized(FileSystem fs) throws IOException { + public static boolean isInitialized(VolumeManager fs) throws IOException { return (fs.exists(ServerConstants.getInstanceIdLocation()) || fs.exists(ServerConstants.getDataVersionLocation())); } @@ -505,7 +506,7 @@ public class Initialize { Configuration conf = CachedConfiguration.getInstance(); @SuppressWarnings("deprecation") - FileSystem fs = FileSystemImpl.get(SiteConfiguration.getSiteConfiguration()); + VolumeManager fs = VolumeManagerImpl.get(SiteConfiguration.getSiteConfiguration()); if (opts.resetSecurity) { if (isInitialized(fs)) { Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java Tue Jun 18 17:45:01 2013 @@ -29,10 +29,11 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.cli.ClientOpts; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class LocalityCheck { @@ -41,7 +42,7 @@ public class LocalityCheck { ClientOpts opts = new ClientOpts(); opts.parseArgs(LocalityCheck.class.getName(), args); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); Connector connector = opts.getConnector(); Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY); @@ -72,7 +73,7 @@ public class LocalityCheck { return 0; } - private void addBlocks(FileSystem fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { + private void addBlocks(VolumeManager fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; if (!totalBlocks.containsKey(host)) { @@ -81,7 +82,7 @@ public class LocalityCheck { } for (String file : files) { Path filePath = new Path(file); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(filePath); + FileSystem ns = fs.getFileSystemByPath(filePath); FileStatus fileStatus = ns.getFileStatus(filePath); BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Tue Jun 18 17:45:01 2013 @@ -70,8 +70,8 @@ import org.apache.accumulo.fate.zookeepe import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.accumulo.server.fs.FileRef; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.security.SecurityConstants; import org.apache.accumulo.server.zookeeper.ZooLock; @@ -341,7 +341,7 @@ public class MetadataTable extends org.a Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Authorizations.EMPTY); mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY); Text row = extent.getMetadataEntry(); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text("")); endKey = endKey.followingKey(PartialKey.ROW_COLFAM); @@ -491,7 +491,7 @@ public class MetadataTable extends org.a pathToRemove = "/" + tableId + "/" + pathToRemove; } - Path path = FileSystemImpl.get().getFullPath(ServerConstants.getTablesDirs(), pathToRemove); + Path path = VolumeManagerImpl.get().getFullPath(ServerConstants.getTablesDirs(), pathToRemove); Mutation delFlag = new Mutation(new Text(prefix + path.toString())); delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {})); return delFlag; @@ -519,7 +519,7 @@ public class MetadataTable extends org.a ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Authorizations.EMPTY); scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW))); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); if (!scanner2.iterator().hasNext()) { log.info("Rolling back incomplete split " + metadataEntry + " " + metadataPrevEndRow); rollBackSplit(metadataEntry, KeyExtent.decodePrevEndRow(oper), credentials, lock); @@ -658,7 +658,7 @@ public class MetadataTable extends org.a Key key = cell.getKey(); if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) { - FileRef ref = new FileRef(FileSystemImpl.get(), key); + FileRef ref = new FileRef(VolumeManagerImpl.get(), key); bw.addMutation(createDeleteMutation(tableId, ref.meta().toString())); } @@ -796,11 +796,11 @@ public class MetadataTable extends org.a ArrayList<LogEntry> result = new ArrayList<LogEntry>(); TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>(); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); if (extent.isRootTablet()) { getRootLogEntries(result); Path rootDir = new Path(ServerConstants.getRootTabletDir()); - rootDir = rootDir.makeQualified(fs.getDefaultNamespace()); + rootDir = rootDir.makeQualified(fs.getDefaultVolume()); FileStatus[] files = fs.listStatus(rootDir); for (FileStatus fileStatus : files) { if (fileStatus.getPath().toString().endsWith("_tmp")) { @@ -1191,7 +1191,7 @@ public class MetadataTable extends org.a public static List<FileRef> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) throws IOException { List<FileRef> result = new ArrayList<FileRef>(); try { - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Authorizations.EMPTY)); mscanner.setRange(extent.toMetadataRange()); mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY); @@ -1215,7 +1215,7 @@ public class MetadataTable extends org.a Map<FileRef,Long> ret = new HashMap<FileRef,Long>(); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Authorizations.EMPTY); scanner.setRange(new Range(metadataRow)); scanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY); Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java Tue Jun 18 17:45:01 2013 @@ -51,23 +51,24 @@ import org.apache.accumulo.core.util.Tex import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.util.MetadataTable.LogEntry; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class OfflineMetadataScanner extends ScannerOptions implements Scanner { private Set<String> allFiles = new HashSet<String>(); private Range range = new Range(); - private final FileSystem fs; + private final VolumeManager fs; private final AccumuloConfiguration conf; - private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String> files, FileSystem fs, AccumuloConfiguration conf) throws IOException { + private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String> files, VolumeManager fs, AccumuloConfiguration conf) throws IOException { List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(); for (String file : files) { - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(file); + FileSystem ns = fs.getFileSystemByPath(file); FileSKVIterator reader = FileOperations.getInstance().openReader(file, true, ns, ns.getConf(), conf); readers.add(reader); } @@ -119,7 +120,7 @@ public class OfflineMetadataScanner exte } - public OfflineMetadataScanner(AccumuloConfiguration conf, FileSystem fs) throws IOException { + public OfflineMetadataScanner(AccumuloConfiguration conf, VolumeManager fs) throws IOException { super(); this.fs = fs; this.conf = conf; @@ -258,7 +259,7 @@ public class OfflineMetadataScanner exte public static void main(String[] args) throws IOException { ServerConfiguration conf = new ServerConfiguration(HdfsZooInstance.getInstance()); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); OfflineMetadataScanner scanner = new OfflineMetadataScanner(conf.getConfiguration(), fs); scanner.setRange(Constants.METADATA_KEYSPACE); for (Entry<Key,Value> entry : scanner) Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java Tue Jun 18 17:45:01 2013 @@ -28,8 +28,8 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.server.cli.ClientOpts; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; @@ -52,7 +52,7 @@ public class RemoveEntriesForMissingFile ScannerOpts scanOpts = new ScannerOpts(); BatchWriterOpts bwOpts = new BatchWriterOpts(); opts.parseArgs(RemoveEntriesForMissingFiles.class.getName(), args, scanOpts, bwOpts); - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); Connector connector = opts.getConnector(); Scanner metadata = connector.createScanner(Constants.METADATA_TABLE_NAME, opts.auths); metadata.setBatchSize(scanOpts.scanBatchSize); Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java Tue Jun 18 17:45:01 2013 @@ -22,8 +22,8 @@ import java.util.Random; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.server.ServerConstants; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.tabletserver.UniqueNameAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -36,7 +36,7 @@ public class TabletOperations { private static final Random random = new Random(); // TODO ACCUMULO-118 make the namespace selection pluggable - public static String createTabletDirectory(FileSystem fs, String tableId, Text endRow) { + public static String createTabletDirectory(VolumeManager fs, String tableId, Text endRow) { String lowDirectory; UniqueNameAllocator namer = UniqueNameAllocator.getInstance(); @@ -72,7 +72,7 @@ public class TabletOperations { public static String createTabletDirectory(String tableDir, Text endRow) { while (true) { try { - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); return createTabletDirectory(fs, tableDir, endRow); } catch (IOException e) { log.warn(e); Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java Tue Jun 18 17:45:01 2013 @@ -35,8 +35,8 @@ import org.apache.accumulo.core.data.Val import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.thrift.TCredentials; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.hadoop.io.Text; import org.junit.Assert; import org.junit.Test; @@ -94,7 +94,7 @@ public class TestConfirmDeletes { TCredentials auth = CredentialHelper.create("root", new PasswordToken(new byte[0]), "instance"); Instance instance = new MockInstance(); - FileSystem fs = FileSystemImpl.getLocal(); + VolumeManager fs = VolumeManagerImpl.getLocal(); load(instance, metadata, deletes); Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java Tue Jun 18 17:45:01 2013 @@ -22,8 +22,9 @@ import static org.junit.Assert.assertTru import java.io.IOException; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; @@ -37,20 +38,20 @@ import org.junit.rules.TemporaryFolder; public class MultiReaderTest { - FileSystem fs; + VolumeManager fs; TemporaryFolder root = new TemporaryFolder(); @Before public void setUp() throws Exception { // quiet log messages about compress.CodecPool Logger.getRootLogger().setLevel(Level.ERROR); - fs = FileSystemImpl.getLocal(); + fs = VolumeManagerImpl.getLocal(); root.create(); String path = root.getRoot().getAbsolutePath(); Path root = new Path("file://" + path + "/manyMaps"); fs.mkdirs(root); fs.create(new Path(root, "finished")).close(); - org.apache.hadoop.fs.FileSystem ns = fs.getDefaultNamespace(); + FileSystem ns = fs.getDefaultVolume(); Writer writer = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class); BytesWritable value = new BytesWritable("someValue".getBytes()); for (int i = 1; i < 1000; i += 2) { Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java Tue Jun 18 17:45:01 2013 @@ -37,11 +37,12 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.server.data.ServerMutation; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.logger.LogEvents; import org.apache.accumulo.server.logger.LogFileKey; import org.apache.accumulo.server.logger.LogFileValue; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.MapFile.Writer; @@ -116,13 +117,13 @@ public class SortedLogRecoveryTest { TemporaryFolder root = new TemporaryFolder(); root.create(); final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir"; - FileSystem fs = FileSystemImpl.getLocal(); + VolumeManager fs = VolumeManagerImpl.getLocal(); fs.deleteRecursively(new Path(workdir)); ArrayList<Path> dirs = new ArrayList<Path>(); try { for (Entry<String,KeyValue[]> entry : logs.entrySet()) { String path = workdir + "/" + entry.getKey(); - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getFileSystemByPath(path); Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class); for (KeyValue lfe : entry.getValue()) { map.append(lfe.key, lfe.value); Modified: accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousStatsCollector.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousStatsCollector.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousStatsCollector.java (original) +++ accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousStatsCollector.java Tue Jun 18 17:45:01 2013 @@ -41,13 +41,14 @@ import org.apache.accumulo.core.util.Cac import org.apache.accumulo.core.util.Stat; import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.cli.ClientOnRequiredTable; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.monitor.Monitor; import org.apache.accumulo.server.security.SecurityConstants; import org.apache.accumulo.trace.instrument.Tracer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobClient; @@ -110,15 +111,15 @@ public class ContinuousStatsCollector { } private String getFSStats() throws Exception { - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); long length1 = 0, dcount1 = 0, fcount1 = 0; long length2 = 0, dcount2 = 0, fcount2 = 0; for (String dir : ServerConstants.getTablesDirs()) { - ContentSummary contentSummary = fs.getContentSummary(dir); + ContentSummary contentSummary = fs.getContentSummary(new Path(dir)); length1 += contentSummary.getLength(); dcount1 += contentSummary.getDirectoryCount(); fcount1 += contentSummary.getFileCount(); - contentSummary = fs.getContentSummary(dir + "/" + tableId); + contentSummary = fs.getContentSummary(new Path(dir, tableId)); length2 += contentSummary.getLength(); dcount2 += contentSummary.getDirectoryCount(); fcount2 += contentSummary.getFileCount(); Modified: accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java?rev=1494226&r1=1494225&r2=1494226&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java (original) +++ accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java Tue Jun 18 17:45:01 2013 @@ -68,11 +68,12 @@ import org.apache.accumulo.server.cli.Cl import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.fs.FileRef; -import org.apache.accumulo.server.fs.FileSystem; -import org.apache.accumulo.server.fs.FileSystemImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.util.MetadataTable; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; @@ -104,7 +105,7 @@ public class CollectTabletStats { columnsTmp = opts.columns.split(","); final String columns[] = columnsTmp; - final FileSystem fs = FileSystemImpl.get(); + final VolumeManager fs = VolumeManagerImpl.get(); Instance instance = opts.getInstance(); final ServerConfiguration sconf = new ServerConfiguration(instance); @@ -376,7 +377,7 @@ public class CollectTabletStats { } private static void reportHdfsBlockLocations(List<FileRef> files) throws Exception { - FileSystem fs = FileSystemImpl.get(); + VolumeManager fs = VolumeManagerImpl.get(); System.out.println("\t\tFile block report : "); for (FileRef file : files) { @@ -386,7 +387,7 @@ public class CollectTabletStats { // assume it is a map file status = fs.getFileStatus(new Path(file + "/data")); } - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getFileSystemByPath(file.path()); BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen()); System.out.println("\t\t\tBlocks for : " + file); @@ -426,14 +427,14 @@ public class CollectTabletStats { return visFilter; } - private static int readFiles(FileSystem fs, AccumuloConfiguration aconf, List<FileRef> files, KeyExtent ke, String[] columns) throws Exception { + private static int readFiles(VolumeManager fs, AccumuloConfiguration aconf, List<FileRef> files, KeyExtent ke, String[] columns) throws Exception { int count = 0; HashSet<ByteSequence> columnSet = createColumnBSS(columns); for (FileRef file : files) { - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getFileSystemByPath(file.path()); FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf); Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true); reader.seek(range, columnSet, columnSet.size() == 0 ? false : true); @@ -455,7 +456,7 @@ public class CollectTabletStats { return columnSet; } - private static int readFilesUsingIterStack(FileSystem fs, ServerConfiguration aconf, List<FileRef> files, Authorizations auths, KeyExtent ke, String[] columns, + private static int readFilesUsingIterStack(VolumeManager fs, ServerConfiguration aconf, List<FileRef> files, Authorizations auths, KeyExtent ke, String[] columns, boolean useTableIterators) throws Exception { @@ -464,7 +465,7 @@ public class CollectTabletStats { List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(files.size()); for (FileRef file : files) { - org.apache.hadoop.fs.FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getFileSystemByPath(file.path()); readers.add(FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf.getConfiguration())); }
