Repository: hbase Updated Branches: refs/heads/hbase-14439 f2856e2c5 -> e7743d779
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java index 630ca78..be820a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java @@ -25,12 +25,14 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; /** * Thread that walks over the filesystem, and computes the mappings @@ -41,12 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils; class FSRegionScanner implements Runnable { static private final Log LOG = LogFactory.getLog(FSRegionScanner.class); - private Path regionPath; - - /** - * The file system used - */ - private FileSystem fs; + private HRegionFileSystem rfs; /** * Maps each region to the RS with highest locality for that region. @@ -59,11 +56,10 @@ class FSRegionScanner implements Runnable { */ private Map<String, Map<String, Float>> regionDegreeLocalityMapping; - FSRegionScanner(FileSystem fs, Path regionPath, + FSRegionScanner(Configuration conf, HRegionInfo hri, Map<String, String> regionToBestLocalityRSMapping, Map<String, Map<String, Float>> regionDegreeLocalityMapping) { - this.fs = fs; - this.regionPath = regionPath; + this.rfs = HRegionFileSystem.open(conf, hri); this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping; this.regionDegreeLocalityMapping = regionDegreeLocalityMapping; } @@ -75,33 +71,13 @@ class FSRegionScanner implements Runnable { Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>(); //get table name - String tableName = regionPath.getParent().getName(); + String tableName = rfs.getTable(); int totalBlkCount = 0; - // ignore null - FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs)); - if (null == cfList) { - return; - } - - // for each cf, get all the blocks information - for (FileStatus cfStatus : cfList) { - if (!cfStatus.isDirectory()) { - // skip because this is not a CF directory - continue; - } - - FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath()); - if (null == storeFileLists) { - continue; - } - - for (FileStatus storeFile : storeFileLists) { - BlockLocation[] blkLocations = - fs.getFileBlockLocations(storeFile, 0, storeFile.getLen()); - if (null == blkLocations) { - continue; - } + for (String family: rfs.getFamilies()) { + for (StoreFileInfo storeFile: rfs.getStoreFiles(family)) { + BlockLocation[] blkLocations = storeFile.getFileBlockLocations(fs); + if (blkLocations == null) continue; totalBlkCount += blkLocations.length; for(BlockLocation blk: blkLocations) { @@ -138,7 +114,7 @@ class FSRegionScanner implements Runnable { if (hostToRun.endsWith(".")) { hostToRun = hostToRun.substring(0, hostToRun.length()-1); } - String name = tableName + ":" + regionPath.getName(); + String name = tableName + ":" + rfs.getRegionInfo().getEncodedName(); synchronized (regionToBestLocalityRSMapping) { regionToBestLocalityRSMapping.put(name, hostToRun); } @@ -157,7 +133,7 @@ class FSRegionScanner implements Runnable { } // Put the locality map into the result map, keyed by the encoded name // of the region. - regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap); + regionDegreeLocalityMapping.put(rfs.getRegionInfo().getEncodedName(), hostLocalityMap); } } catch (IOException e) { LOG.warn("Problem scanning file system", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index cce37d7..426b73e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -221,6 +221,7 @@ public class FSTableDescriptors implements TableDescriptors { } else { LOG.debug("Fetching table descriptors from the filesystem."); boolean allvisited = true; + HMasterFileSystem mfs = HMasterFileSystem.open(conf, rootdir); for (Path d : FSUtils.getTableDirs(fs, rootdir)) { TableDescriptor htd = null; try { http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index ce51e27..efa8752 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -31,6 +31,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -64,7 +65,8 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; -import org.apache.hadoop.hbase.fs.layout.FsLayout; +import org.apache.hadoop.hbase.fs.HMasterFileSystem; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -1027,43 +1029,6 @@ public abstract class FSUtils { } - - /** - * Runs through the hbase rootdir and checks all stores have only - * one file in them -- that is, they've been major compacted. Looks - * at root and meta tables too. - * @param fs filesystem - * @param hbaseRootDir hbase root directory - * @return True if this hbase install is major compacted. - * @throws IOException e - */ - public static boolean isMajorCompacted(final FileSystem fs, - final Path hbaseRootDir) - throws IOException { - List<Path> tableDirs = getTableDirs(fs, hbaseRootDir); - RegionDirFilter regionFilter = new RegionDirFilter(fs); - PathFilter familyFilter = new FamilyDirFilter(fs); - for (Path tableDir : tableDirs) { - List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir, regionFilter); - for (FileStatus regionDir : regionDirs) { - Path dd = regionDir.getPath(); - // Else its a region name. Now look in region for families. - FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); - for (FileStatus familyDir : familyDirs) { - Path family = familyDir.getPath(); - // Now in family make sure only one file. - FileStatus[] familyStatus = fs.listStatus(family); - if (familyStatus.length > 1) { - LOG.debug(family.toString() + " has " + familyStatus.length + - " files."); - return false; - } - } - } - } - return true; - } - // TODO move this method OUT of FSUtils. No dependencies to HMaster /** * Returns the total overall fragmentation percentage. Includes hbase:meta and @@ -1109,38 +1074,32 @@ public abstract class FSUtils { * @throws IOException When scanning the directory fails. */ public static Map<String, Integer> getTableFragmentation( - final FileSystem fs, final Path hbaseRootDir) - throws IOException { - Map<String, Integer> frags = new HashMap<String, Integer>(); - int cfCountTotal = 0; + final FileSystem fs, final Path hbaseRootDir) throws IOException { + final Map<String, Integer> frags = new HashMap<String, Integer>(); + int cfFragTotal = 0; - RegionDirFilter regionFilter = new RegionDirFilter(fs); - PathFilter familyFilter = new FamilyDirFilter(fs); - List<Path> tableDirs = getTableDirs(fs, hbaseRootDir); - for (Path tableDir : tableDirs) { + int cfCountTotal = 0; + HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir); + for (TableName tableName: mfs.getTables()) { int cfCount = 0; int cfFrag = 0; - List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir, regionFilter); - for (FileStatus regionDir : regionDirs) { - Path dd = regionDir.getPath(); - // else its a region name, now look in region for families - FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); - for (FileStatus familyDir : familyDirs) { + for (HRegionInfo hri: mfs.getRegions(tableName)) { + HRegionFileSystem rfs = HRegionFileSystem.open(conf, hri); + for (String family: rfs.getFamilies()) { cfCount++; cfCountTotal++; - Path family = familyDir.getPath(); - // now in family make sure only one file - FileStatus[] familyStatus = fs.listStatus(family); - if (familyStatus.length > 1) { + Collection<StoreFileInfo> storeFiles = rfs.getStoreFiles(family); + if (storeFiles.size() > 1) { cfFrag++; cfFragTotal++; } } } // compute percentage per table and store in result list - frags.put(FSUtils.getTableName(tableDir).getNameAsString(), + frags.put(tableName.getNameAsString(), cfCount == 0? 0: Math.round((float) cfFrag / cfCount * 100)); } + // set overall percentage for all tables frags.put("-TOTAL-", cfCountTotal == 0? 0: Math.round((float) cfFragTotal / cfCountTotal * 100)); @@ -1259,29 +1218,6 @@ public abstract class FSUtils { } /** - * A {@link PathFilter} that returns usertable directories. To get all directories use the - * {@link BlackListDirFilter} with a <tt>null</tt> blacklist - */ - public static class UserTableDirFilter extends BlackListDirFilter { - public UserTableDirFilter(FileSystem fs) { - super(fs, HConstants.HBASE_NON_TABLE_DIRS); - } - - protected boolean isValidName(final String name) { - if (!super.isValidName(name)) - return false; - - try { - TableName.isLegalTableQualifierName(Bytes.toBytes(name)); - } catch (IllegalArgumentException e) { - LOG.info("INVALID NAME " + name); - return false; - } - return true; - } - } - - /** * Heuristic to determine whether is safe or not to open a file for append * Looks both for dfs.support.append and use reflection to search * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush() @@ -1336,36 +1272,6 @@ public abstract class FSUtils { public abstract void recoverFileLease(final FileSystem fs, final Path p, Configuration conf, CancelableProgressable reporter) throws IOException; - public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir) - throws IOException { - List<Path> tableDirs = new LinkedList<Path>(); - - for(FileStatus status : - fs.globStatus(new Path(rootdir, - new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) { - tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath())); - } - return tableDirs; - } - - /** - * @param fs - * @param rootdir - * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as - * .logs, .oldlogs, .corrupt folders. - * @throws IOException - */ - public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir) - throws IOException { - // presumes any directory under hbase.rootdir is a table - FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs)); - List<Path> tabledirs = new ArrayList<Path>(dirs.length); - for (FileStatus dir: dirs) { - tabledirs.add(dir.getPath()); - } - return tabledirs; - } - /** * Checks if the given path is the one with 'recovered.edits' dir. * @param path @@ -1376,94 +1282,6 @@ public abstract class FSUtils { } /** - * Filter for all dirs that don't start with '.' - */ - public static class RegionDirFilter implements PathFilter { - // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names. - final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$"); - final FileSystem fs; - - public RegionDirFilter(FileSystem fs) { - this.fs = fs; - } - - @Override - public boolean accept(Path rd) { - if (!regionDirPattern.matcher(rd.getName()).matches()) { - return false; - } - - try { - return fs.getFileStatus(rd).isDirectory(); - } catch (IOException ioe) { - // Maybe the file was moved or the fs was disconnected. - LOG.warn("Skipping file " + rd +" due to IOException", ioe); - return false; - } - } - } - - /** - * Filter for all dirs that are legal column family names. This is generally used for colfam - * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>. - */ - public static class FamilyDirFilter implements PathFilter { - final FileSystem fs; - - public FamilyDirFilter(FileSystem fs) { - this.fs = fs; - } - - @Override - public boolean accept(Path rd) { - try { - // throws IAE if invalid - HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName())); - } catch (IllegalArgumentException iae) { - // path name is an invalid family name and thus is excluded. - return false; - } - - try { - return fs.getFileStatus(rd).isDirectory(); - } catch (IOException ioe) { - // Maybe the file was moved or the fs was disconnected. - LOG.warn("Skipping file " + rd +" due to IOException", ioe); - return false; - } - } - } - - /** - * Given a particular region dir, return all the familydirs inside it - * - * @param fs A file system for the Path - * @param regionDir Path to a specific region directory - * @return List of paths to valid family directories in region dir. - * @throws IOException - */ - public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException { - // assumes we are in a region dir. - FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs)); - List<Path> familyDirs = new ArrayList<Path>(fds.length); - for (FileStatus fdfs: fds) { - Path fdPath = fdfs.getPath(); - familyDirs.add(fdPath); - } - return familyDirs; - } - - public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir) throws IOException { - FileStatus[] fds = fs.listStatus(familyDir, new ReferenceFileFilter(fs)); - List<Path> referenceFiles = new ArrayList<Path>(fds.length); - for (FileStatus fdfs: fds) { - Path fdPath = fdfs.getPath(); - referenceFiles.add(fdPath); - } - return referenceFiles; - } - - /** * Filter for HFiles that excludes reference files. */ public static class HFileFilter implements PathFilter { @@ -1514,7 +1332,7 @@ public abstract class FSUtils { * @throws IOException */ public static FileSystem getCurrentFileSystem(Configuration conf) - throws IOException { + throws IOException { return getRootDir(conf).getFileSystem(conf); } @@ -1557,59 +1375,36 @@ public abstract class FSUtils { * @throws IOException When scanning the directory fails. */ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map, - final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors) - throws IOException { + final FileSystem fs, final Path hbaseRootDir, final TableName tableName, + final ErrorReporter errors) throws IOException { + HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir); + return getTableStoreFilePathMap(map, mfs, tableName, errors); + } + + private static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map, + final HMasterFileSystem mfs, final TableName tableName, final ErrorReporter errors) + throws IOException { if (map == null) { map = new HashMap<String, Path>(); } + for (HRegionInfo hri: mfs.getRegions(tableName)) { + if (errors != null) errors.progress(); - // only include the directory paths to tables - Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName); - // Inside a table, there are compaction.dir directories to skip. Otherwise, all else - // should be regions. - PathFilter familyFilter = new FamilyDirFilter(fs); - List<Path> regionDirs = FsLayout.getRegionDirPaths(fs, tableDir); - for (Path dd : regionDirs) { - if (null != errors) { - errors.progress(); - } - // else its a region name, now look in region for families - FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); - for (FileStatus familyDir : familyDirs) { - if (null != errors) { - errors.progress(); - } - Path family = familyDir.getPath(); - if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) { - continue; - } - // now in family, iterate over the StoreFiles and - // put in map - FileStatus[] familyStatus = fs.listStatus(family); - for (FileStatus sfStatus : familyStatus) { - if (null != errors) { - errors.progress(); - } - Path sf = sfStatus.getPath(); - map.put( sf.getName(), sf); + HRegionFileSystem rfs = HRegionFileSystem.open(conf, hri); + for (String family: rfs.getFamilies()) { + if (errors != null) errors.progress(); + + for (StoreFileInfo storeFile: rfs.getStoreFiles(family)) { + if (errors != null) errors.progress(); + + Path sf = storeFile.getPath(); + map.put(sf.getName(), sf); } } } return map; } - public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) { - int result = 0; - try { - for (Path familyDir:getFamilyDirs(fs, p)){ - result += getReferenceFilePaths(fs, familyDir).size(); - } - } catch (IOException e) { - LOG.warn("Error Counting reference files.", e); - } - return result; - } - /** * Runs through the HBase rootdir and creates a reverse lookup map for * table StoreFile names to the full Path. @@ -1643,18 +1438,13 @@ public abstract class FSUtils { * @return Map keyed by StoreFile name with a value of the full Path. * @throws IOException When scanning the directory fails. */ - public static Map<String, Path> getTableStoreFilePathMap( - final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors) - throws IOException { - Map<String, Path> map = new HashMap<String, Path>(); - - // if this method looks similar to 'getTableFragmentation' that is because - // it was borrowed from it. - - // only include the directory paths to tables - for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) { - getTableStoreFilePathMap(map, fs, hbaseRootDir, - FSUtils.getTableName(tableDir), errors); + public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs, + final Path hbaseRootDir, ErrorReporter errors) throws IOException { + final Map<String, Path> map = new HashMap<String, Path>(); + HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir); + for (TableName tableName: mfs.getTables()) { + if (errors != null) errors.progress(); + getTableStoreFilePathMap(map, mfs, tableName, errors); } return map; } @@ -1875,74 +1665,34 @@ public abstract class FSUtils { FileSystem fs = FileSystem.get(conf); Path rootPath = FSUtils.getRootDir(conf); long startTime = EnvironmentEdgeManager.currentTime(); - Path queryPath; - // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/* - if (null == desiredTable) { - queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/"); - } else { - queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/"); - } - // reject all paths that are not appropriate - PathFilter pathFilter = new PathFilter() { - @Override - public boolean accept(Path path) { - // this is the region name; it may get some noise data - if (null == path) { - return false; - } - // no parent? - Path parent = path.getParent(); - if (null == parent) { - return false; - } - - String regionName = path.getName(); - if (null == regionName) { - return false; - } - - if (!regionName.toLowerCase().matches("[0-9a-f]+")) { - return false; - } - return true; + HMasterFileSystem mfs = HMasterFileSystem.open(conf); + List<HRegionInfo> hris; + if (desiredTable != null) { + hris = mfs.getRegions(TableName.valueOf(desiredTable)); + } else { + hris = new ArrayList<HRegionInfo>(); + for (TableName tableName: mfs.getTables()) { + hris.addAll(mfs.getRegions(tableName)); } - }; - - FileStatus[] statusList = fs.globStatus(queryPath, pathFilter); + } - if (null == statusList) { + if (hris.isEmpty()) { return; - } else { - LOG.debug("Query Path: " + queryPath + " ; # list of files: " + - statusList.length); } // lower the number of threads in case we have very few expected regions - threadPoolSize = Math.min(threadPoolSize, statusList.length); + threadPoolSize = Math.min(threadPoolSize, hris.size()); // run in multiple threads ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize, threadPoolSize, 60, TimeUnit.SECONDS, - new ArrayBlockingQueue<Runnable>(statusList.length)); + new ArrayBlockingQueue<Runnable>(hris.size())); try { // ignore all file status items that are not of interest - for (FileStatus regionStatus : statusList) { - if (null == regionStatus) { - continue; - } - - if (!regionStatus.isDirectory()) { - continue; - } - - Path regionPath = regionStatus.getPath(); - if (null == regionPath) { - continue; - } - - tpe.execute(new FSRegionScanner(fs, regionPath, + for (HRegionInfo hri: hris) { + tpe.execute(new FSRegionScanner(conf, hri, regionToBestLocalityRSMapping, regionDegreeLocalityMapping)); } } finally { @@ -2010,7 +1760,7 @@ public abstract class FSUtils { /** * @param c * @return The DFSClient DFSHedgedReadMetrics instance or null if can't be found or not on hdfs. - * @throws IOException + * @throws IOException */ public static DFSHedgedReadMetrics getDFSHedgedReadMetrics(final Configuration c) throws IOException { http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java deleted file mode 100644 index efd8124..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java +++ /dev/null @@ -1,231 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.io.IOException; -import java.util.List; -import java.util.NavigableSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.fs.layout.FsLayout; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.wal.WALSplitter; - -/** - * Utility methods for interacting with the hbase.root file system. - */ [email protected] -public final class FSVisitor { - private static final Log LOG = LogFactory.getLog(FSVisitor.class); - - public interface RegionVisitor { - void region(final String region) throws IOException; - } - - public interface StoreFileVisitor { - void storeFile(final String region, final String family, final String hfileName) - throws IOException; - } - - public interface RecoveredEditsVisitor { - void recoveredEdits (final String region, final String logfile) - throws IOException; - } - - public interface LogFileVisitor { - void logFile (final String server, final String logfile) - throws IOException; - } - - private FSVisitor() { - // private constructor for utility class - } - - /** - * Iterate over the table store files - * - * @param fs {@link FileSystem} - * @param tableDir {@link Path} to the table directory - * @param visitor callback object to get the store files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitRegions(final FileSystem fs, final Path tableDir, - final RegionVisitor visitor) throws IOException { - List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs)); - if (regions == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No regions under directory:" + tableDir); - } - return; - } - - for (FileStatus region: regions) { - visitor.region(region.getPath().getName()); - } - } - - /** - * Iterate over the table store files - * - * @param fs {@link FileSystem} - * @param tableDir {@link Path} to the table directory - * @param visitor callback object to get the store files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir, - final StoreFileVisitor visitor) throws IOException { - List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs)); - if (regions == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No regions under directory:" + tableDir); - } - return; - } - - for (FileStatus region: regions) { - visitRegionStoreFiles(fs, region.getPath(), visitor); - } - } - - /** - * Iterate over the region store files - * - * @param fs {@link FileSystem} - * @param regionDir {@link Path} to the region directory - * @param visitor callback object to get the store files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir, - final StoreFileVisitor visitor) throws IOException { - FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs)); - if (families == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No families under region directory:" + regionDir); - } - return; - } - - PathFilter fileFilter = new FSUtils.FileFilter(fs); - for (FileStatus family: families) { - Path familyDir = family.getPath(); - String familyName = familyDir.getName(); - - // get all the storeFiles in the family - FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter); - if (storeFiles == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No hfiles found for family: " + familyDir + ", skipping."); - } - continue; - } - - for (FileStatus hfile: storeFiles) { - Path hfilePath = hfile.getPath(); - visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName()); - } - } - } - - /** - * Iterate over each region in the table and inform about recovered.edits - * - * @param fs {@link FileSystem} - * @param tableDir {@link Path} to the table directory - * @param visitor callback object to get the recovered.edits files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir, - final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { - List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs)); - if (regions == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No recoveredEdits regions under directory:" + tableDir); - } - return; - } - - for (FileStatus region: regions) { - visitRegionRecoveredEdits(fs, region.getPath(), visitor); - } - } - - /** - * Iterate over recovered.edits of the specified region - * - * @param fs {@link FileSystem} - * @param regionDir {@link Path} to the Region directory - * @param visitor callback object to get the recovered.edits files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir, - final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { - NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regionDir); - if (files == null || files.size() == 0) return; - - for (Path source: files) { - // check to see if the file is zero length, in which case we can skip it - FileStatus stat = fs.getFileStatus(source); - if (stat.getLen() <= 0) continue; - - visitor.recoveredEdits(regionDir.getName(), source.getName()); - } - } - - /** - * Iterate over hbase log files - * - * @param fs {@link FileSystem} - * @param rootDir {@link Path} to the HBase root folder - * @param visitor callback object to get the log files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitLogFiles(final FileSystem fs, final Path rootDir, - final LogFileVisitor visitor) throws IOException { - Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir); - if (logServerDirs == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No logs under directory:" + logsDir); - } - return; - } - - for (FileStatus serverLogs: logServerDirs) { - String serverName = serverLogs.getPath().getName(); - - FileStatus[] wals = FSUtils.listStatus(fs, serverLogs.getPath()); - if (wals == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No wals found for server: " + serverName + ", skipping."); - } - continue; - } - - for (FileStatus walRef: wals) { - visitor.logFile(serverName, walRef.getPath().getName()); - } - } - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 98ad22b..8b2796e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.fs.layout.FsLayout; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.master.MasterFileSystem; @@ -118,7 +119,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; @@ -395,7 +395,7 @@ public class HBaseFsck extends Configured implements Closeable { LOG.info("Failed to create lock file " + hbckLockFilePath.getName() + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts()); - LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), + LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), ioe); try { exception = ioe; @@ -762,7 +762,7 @@ public class HBaseFsck extends Configured implements Closeable { currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); // For each region, get the start and stop key from the META and compare them to the // same information from the Stores. - HRegionFileSystem hrfs = HRegionFileSystem.create(getConf(), fs, tableDir, regionInfo); + HRegionFileSystem hrfs = HRegionFileSystem.open(getConf(), regionInfo, true); Path path = hrfs.getRegionDir(); FileStatus[] files = fs.listStatus(path); // For all the column families in this region... @@ -2279,8 +2279,7 @@ public class HBaseFsck extends Configured implements Closeable { LOG.warn(hri + " start and stop keys are in the range of " + region + ". The region might not be cleaned up from hdfs when region " + region + " split failed. Hence deleting from hdfs."); - HRegionFileSystem.deleteAndArchiveRegionFromFileSystem(getConf(), fs, - FsLayout.getTableDirFromRegionDir(regionDir), hri); + HRegionFileSystem.destroy(getConf(), hri); return; } } @@ -2688,10 +2687,10 @@ public class HBaseFsck extends Configured implements Closeable { } regionsFromMeta = Ordering.natural().immutableSortedCopy(regions); } - + return regionsFromMeta; } - + private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl { ErrorReporter errors; @@ -4041,7 +4040,7 @@ public class HBaseFsck extends Configured implements Closeable { public synchronized Void call() throws IOException { try { // level 2: <HBASE_DIR>/<table>/* - List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir.getPath(), + List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir.getPath(), new FSUtils.RegionDirFilter(fs)); if (regionDirs == null) { return null; http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java index 7f74d55..f2dbf52 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.io.FileLink; import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java index ea704f8..1b173bb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java @@ -61,7 +61,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.NoServerForRegionException; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -412,7 +412,7 @@ public class RegionSplitter { * Alternative getCurrentNrHRS which is no longer available. * @param connection * @return Rough count of regionservers out on cluster. - * @throws IOException + * @throws IOException */ private static int getRegionServerCount(final Connection connection) throws IOException { try (Admin admin = connection.getAdmin()) { @@ -767,7 +767,7 @@ public class RegionSplitter { * @param conf * @param tableName * @return A Pair where first item is table dir and second is the split file. - * @throws IOException + * @throws IOException */ private static Pair<Path, Path> getTableDirAndSplitFile(final Configuration conf, final TableName tableName) @@ -785,7 +785,7 @@ public class RegionSplitter { getTableDirAndSplitFile(connection.getConfiguration(), tableName); Path tableDir = tableDirAndSplitFile.getFirst(); Path splitFile = tableDirAndSplitFile.getSecond(); - + FileSystem fs = tableDir.getFileSystem(connection.getConfiguration()); // Using strings because (new byte[]{0}).equals(new byte[]{0}) == false http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java index 303ed60..9cbb135 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java @@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.fs.layout.FsLayout; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -43,10 +42,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CorruptHFileException; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter; -import org.apache.hadoop.hbase.util.FSUtils.HFileFilter; -import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java deleted file mode 100644 index 7aa9761..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.UUID; -import java.util.Set; -import java.util.HashSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.wal.WALSplitter; -import org.apache.hadoop.hbase.fs.layout.FsLayout; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.junit.*; -import org.junit.experimental.categories.Category; - -/** - * Test {@link FSUtils}. - */ -@Category({MiscTests.class, MediumTests.class}) -public class TestFSVisitor { - private static final Log LOG = LogFactory.getLog(TestFSVisitor.class); - - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - private final String TABLE_NAME = "testtb"; - - private Set<String> tableFamilies; - private Set<String> tableRegions; - private Set<String> recoveredEdits; - private Set<String> tableHFiles; - private Set<String> regionServers; - private Set<String> serverLogs; - - private FileSystem fs; - private Path tableDir; - private Path logsDir; - private Path rootDir; - - @Before - public void setUp() throws Exception { - fs = FileSystem.get(TEST_UTIL.getConfiguration()); - rootDir = TEST_UTIL.getDataTestDir("hbase"); - logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - - tableFamilies = new HashSet<String>(); - tableRegions = new HashSet<String>(); - recoveredEdits = new HashSet<String>(); - tableHFiles = new HashSet<String>(); - regionServers = new HashSet<String>(); - serverLogs = new HashSet<String>(); - tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles); - createRecoverEdits(tableDir, tableRegions, recoveredEdits); - createLogs(logsDir, regionServers, serverLogs); - FSUtils.logFileSystemState(fs, rootDir, LOG); - } - - @After - public void tearDown() throws Exception { - fs.delete(rootDir, true); - } - - @Test - public void testVisitStoreFiles() throws IOException { - final Set<String> regions = new HashSet<String>(); - final Set<String> families = new HashSet<String>(); - final Set<String> hfiles = new HashSet<String>(); - FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() { - public void storeFile(final String region, final String family, final String hfileName) - throws IOException { - regions.add(region); - families.add(family); - hfiles.add(hfileName); - } - }); - assertEquals(tableRegions, regions); - assertEquals(tableFamilies, families); - assertEquals(tableHFiles, hfiles); - } - - @Test - public void testVisitRecoveredEdits() throws IOException { - final Set<String> regions = new HashSet<String>(); - final Set<String> edits = new HashSet<String>(); - FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() { - public void recoveredEdits (final String region, final String logfile) - throws IOException { - regions.add(region); - edits.add(logfile); - } - }); - assertEquals(tableRegions, regions); - assertEquals(recoveredEdits, edits); - } - - @Test - public void testVisitLogFiles() throws IOException { - final Set<String> servers = new HashSet<String>(); - final Set<String> logs = new HashSet<String>(); - FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() { - public void logFile (final String server, final String logfile) throws IOException { - servers.add(server); - logs.add(logfile); - } - }); - assertEquals(regionServers, servers); - assertEquals(serverLogs, logs); - } - - - /* - * |-testtb/ - * |----f1d3ff8443297732862df21dc4e57262/ - * |-------f1/ - * |----------d0be84935ba84b66b1e866752ec5d663 - * |----------9fc9d481718f4878b29aad0a597ecb94 - * |-------f2/ - * |----------4b0fe6068c564737946bcf4fd4ab8ae1 - */ - private Path createTableFiles(final Path rootDir, final String tableName, - final Set<String> tableRegions, final Set<String> tableFamilies, - final Set<String> tableHFiles) throws IOException { - Path tableDir = new Path(rootDir, tableName); - for (int r = 0; r < 10; ++r) { - String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r)); - tableRegions.add(regionName); - Path regionDir = FsLayout.getRegionDir(tableDir, regionName); - for (int f = 0; f < 3; ++f) { - String familyName = "f" + f; - tableFamilies.add(familyName); - Path familyDir = new Path(regionDir, familyName); - fs.mkdirs(familyDir); - for (int h = 0; h < 5; ++h) { - String hfileName = UUID.randomUUID().toString().replaceAll("-", ""); - tableHFiles.add(hfileName); - fs.createNewFile(new Path(familyDir, hfileName)); - } - } - } - return tableDir; - } - - /* - * |-testtb/ - * |----f1d3ff8443297732862df21dc4e57262/ - * |-------recovered.edits/ - * |----------0000001351969633479 - * |----------0000001351969633481 - */ - private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions, - final Set<String> recoverEdits) throws IOException { - for (String region: tableRegions) { - Path regionEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(FsLayout.getRegionDir(tableDir, region)); - long seqId = System.currentTimeMillis(); - for (int i = 0; i < 3; ++i) { - String editName = String.format("%019d", seqId + i); - recoverEdits.add(editName); - FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName)); - stream.write(Bytes.toBytes("test")); - stream.close(); - } - } - } - - /* - * Old style - * |-.logs/ - * |----server5,5,1351969633508/ - * |-------server5,5,1351969633508.0 - * |----server6,6,1351969633512/ - * |-------server6,6,1351969633512.0 - * |-------server6,6,1351969633512.3 - * New style - * |-.logs/ - * |----server3,5,1351969633508/ - * |-------server3,5,1351969633508.default.0 - * |----server4,6,1351969633512/ - * |-------server4,6,1351969633512.default.0 - * |-------server4,6,1351969633512.some_provider.3 - */ - private void createLogs(final Path logDir, final Set<String> servers, - final Set<String> logs) throws IOException { - for (int s = 0; s < 7; ++s) { - String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis()); - servers.add(server); - Path serverLogDir = new Path(logDir, server); - if (s % 2 == 0) { - if (s % 3 == 0) { - server += ".default"; - } else { - server += "." + s; - } - } - fs.mkdirs(serverLogDir); - for (int i = 0; i < 5; ++i) { - String logfile = server + '.' + i; - logs.add(logfile); - FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile)); - stream.write(Bytes.toBytes("test")); - stream.close(); - } - } - } -}
