Incomplete experiment (it does not compile and has missing code) Trying to move everyting fs related to fs/ HMasterFileSystem has a LegacyMasterFs and a HierarchicalMasterFs impl HRegionFileSystem has a LegacyRegionFs and a HierarchicalRegionFs impl
I still have the old MasterFileSystem around, because I didn't have time to move it. but the idea is to move everything that loops on the dirs or knows about the layout in the MasterFs and RegionFs classes Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7743d77 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7743d77 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7743d77 Branch: refs/heads/hbase-14439 Commit: e7743d779a8bfcd65d874afdb88d273ba9e5c4f2 Parents: f2856e2 Author: Matteo Bertozzi <[email protected]> Authored: Wed Sep 16 14:12:39 2015 -0700 Committer: Matteo Bertozzi <[email protected]> Committed: Wed Sep 16 16:49:25 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HTableDescriptor.java | 19 - .../hadoop/hbase/fs/HMasterFileSystem.java | 61 + .../hadoop/hbase/fs/HRegionFileSystem.java | 275 ++++ .../hbase/fs/HierarchicalMasterFileSystem.java | 49 + .../hbase/fs/HierarchicalRegionFileSystem.java | 88 ++ .../hadoop/hbase/fs/LegacyMasterFileSystem.java | 152 +++ .../hadoop/hbase/fs/LegacyRegionFileSystem.java | 293 +++++ .../hadoop/hbase/fs/layout/AFsLayout.java | 55 - .../hbase/fs/layout/AHFileLinkManager.java | 73 -- .../apache/hadoop/hbase/fs/layout/FsLayout.java | 220 ---- .../hbase/fs/layout/HierarchicalFsLayout.java | 171 --- .../hbase/fs/layout/StandardHBaseFsLayout.java | 115 -- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 29 +- .../hbase/io/hfile/HFilePrettyPrinter.java | 38 +- .../hadoop/hbase/master/CatalogJanitor.java | 27 +- .../hadoop/hbase/master/MasterFileSystem.java | 40 +- .../hadoop/hbase/master/RegionStates.java | 4 +- .../hbase/regionserver/CompactionTool.java | 2 +- .../hadoop/hbase/regionserver/HRegion.java | 98 +- .../hbase/regionserver/HRegionFileSystem.java | 1216 ------------------ .../regionserver/HRegionFileSystemFactory.java | 30 - .../hadoop/hbase/regionserver/HStore.java | 6 +- .../HierarchicalHRegionFileSystem.java | 48 - .../HierarchicalHRegionFileSystemFactory.java | 31 - .../RegionDoesNotExistException.java | 42 + .../RegionMergeTransactionImpl.java | 3 +- .../regionserver/SplitTransactionImpl.java | 5 +- .../hbase/regionserver/StoreFileInfo.java | 111 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 4 +- .../hadoop/hbase/snapshot/SnapshotManifest.java | 5 +- .../hbase/snapshot/SnapshotManifestV1.java | 8 +- .../hadoop/hbase/util/FSRegionScanner.java | 52 +- .../hadoop/hbase/util/FSTableDescriptors.java | 1 + .../org/apache/hadoop/hbase/util/FSUtils.java | 368 +----- .../org/apache/hadoop/hbase/util/FSVisitor.java | 231 ---- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 15 +- .../hadoop/hbase/util/HFileV1Detector.java | 2 +- .../hadoop/hbase/util/RegionSplitter.java | 8 +- .../hbase/util/hbck/HFileCorruptionChecker.java | 5 - .../apache/hadoop/hbase/util/TestFSVisitor.java | 229 ---- 40 files changed, 1187 insertions(+), 3042 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 9145cdc..1722268 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -1368,25 +1368,6 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> { remove(match); } - /** - * Returns the {@link Path} object representing the table directory under - * path rootdir - * - * Deprecated use FSUtils.getTableDir() instead. - * - * @param rootdir qualified path of HBase root directory - * @param tableName name of table - * @return {@link Path} for table - */ - @Deprecated - public static Path getTableDir(Path rootdir, final byte [] tableName) { - //This is bad I had to mirror code from FSUTils.getTableDir since - //there is no module dependency between hbase-client and hbase-server - TableName name = TableName.valueOf(tableName); - return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR, - new Path(name.getNamespaceAsString(), new Path(name.getQualifierAsString())))); - } - /** Table descriptor for <code>hbase:meta</code> catalog table * Deprecated, use TableDescriptors#get(TableName.META_TABLE) or * Admin#getTableDescriptor(TableName.META_TABLE) instead. http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java new file mode 100644 index 0000000..8f92cf0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java @@ -0,0 +1,61 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.FSUtils; + +public abstract class HMasterFileSystem { + public abstract Collection<String> getNamespaces() throws IOException; + public abstract Collection<TableName> getTables(String namespace) throws IOException; + public abstract Collection<HRegionInfo> getRegions(TableName tableName) throws IOException; + + public Collection<TableName> getTables() throws IOException { + ArrayList<TableName> tables = new ArrayList<TableName>(); + for (String ns: getNamespaces()) { + tables.addAll(getTables(ns)); + } + return tables; + } + + public static HMasterFileSystem open(final Configuration conf) { + return open(conf, FSUtils.getRootDir(conf)); + } + + public static HMasterFileSystem open(Configuration conf, Path rootDir) { + String fsType = conf.get("hbase.fs.layout.type").toLowerCase(); + switch (fsType) { + case "legacy": + return new LegacyMasterFileSystem(conf, rootDir); + case "hierarchical": + return new HierarchicalMasterFileSystem(conf, rootDir); + default: + throw new IOException("Invalid filesystem type " + fsType); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java new file mode 100644 index 0000000..966ac06 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java @@ -0,0 +1,275 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.Bytes; + +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; + [email protected] +public abstract class HRegionFileSystem { + private static final Log LOG = LogFactory.getLog(HRegionFileSystem.class); + + // blah... compat + public enum Type { TABLE, SNAPSHOT, ARCHIVE }; + + /** Name of the region info file that resides just under the region directory. */ + public final static String REGION_INFO_FILE = ".regioninfo"; + + // ========================================================================== + // PUBLIC methods - add/remove/list store files + // ========================================================================== + /** + * Generate a unique temporary Path. Used in conjuction with commitStoreFile() + * to get a safer file creation. + * <code> + * Path file = fs.createTempName(); + * ...StoreFile.Writer(file)... + * fs.commitStoreFile("family", file); + * </code> + * + * @return Unique {@link Path} of the temporary file + */ + public Path createTempName() { + return createTempName(null); + } + + /** + * Generate a unique temporary Path. Used in conjuction with commitStoreFile() + * to get a safer file creation. + * <code> + * Path file = fs.createTempName(); + * ...StoreFile.Writer(file)... + * fs.commitStoreFile("family", file); + * </code> + * + * @param suffix extra information to append to the generated name + * @return Unique {@link Path} of the temporary file + */ + public abstract Path createTempName(final String suffix); // TODO REMOVE THIS + + + /** + * Move the file from a build/temp location to the main family store directory. + * @param familyName Family that will gain the file + * @param buildPath {@link Path} to the file to commit. + * @return The new {@link Path} of the committed file + * @throws IOException + */ + public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException { + return commitStoreFile(familyName, buildPath, -1, false); + } + + public abstract Path commitStoreFile(final String familyName, final Path buildPath, + final long seqNum, final boolean generateNewName) throws IOException; + + /** + * Moves multiple store files to the relative region's family store directory. + * @param storeFiles list of store files divided by family + * @throws IOException + */ + public abstract void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) + throws IOException; + + /** + * Bulk load: Add a specified store file to the specified family. + * If the source file is on the same different file-system is moved from the + * source location to the destination location, otherwise is copied over. + * + * @param familyName Family that will gain the file + * @param srcPath {@link Path} to the file to import + * @param seqNum Bulk Load sequence number + * @return The destination {@link Path} of the bulk loaded file + * @throws IOException + */ + public abstract Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum) + throws IOException; + + /** + * Archives the specified store file from the specified family. + * @param familyName Family that contains the store files + * @param filePath {@link Path} to the store file to remove + * @throws IOException if the archiving fails + */ + public abstract void removeStoreFile(String familyName, Path filePath) + throws IOException; + + /** + * Closes and archives the specified store files from the specified family. + * @param familyName Family that contains the store files + * @param storeFiles set of store files to remove + * @throws IOException if the archiving fails + */ + public abstract void removeStoreFiles(String familyName, Collection<StoreFile> storeFiles) + throws IOException; + + public abstract Path getStoreFilePath(final String familyName, final String fileName); + + /** + * Returns the store files available for the family. + * This methods performs the filtering based on the valid store files. + * @param familyName Column Family Name + * @return a set of {@link StoreFileInfo} for the specified family. + */ + public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException { + return getStoreFiles(Bytes.toString(familyName)); + } + + public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException { + return getStoreFiles(familyName, true); + } + + /** + * Returns the store files available for the family. + * This methods performs the filtering based on the valid store files. + * @param familyName Column Family Name + * @return a set of {@link StoreFileInfo} for the specified family. + */ + public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate) + throws IOException; + + /** + * Return the store file information of the specified family/file. + * + * @param familyName Column Family Name + * @param fileName File Name + * @return The {@link StoreFileInfo} for the specified family/file + */ + public abstract StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) + throws IOException; + + /** + * @return the set of families present on disk + * @throws IOException + */ + public abstract Collection<String> getFamilies() throws IOException; + + // ========================================================================== + // ??? methods - do we still need this stuff in 2.0? + // ========================================================================== + public abstract void writeRecoveryCheckPoint() throws IOException; + public abstract void cleanup() throws IOException; + + // ??? + public void logFileSystemState(Log log) {} + public HRegionInfo getRegionInfoForFS() { return null; } + public FileSystem getFileSystem() { return null; } + public Path getRegionDir() { return null; } + public Path getStoreDir(String family) { return null; } + + // remove on proc-v2 rewrite... move now in Merge/Split FS interface? + public void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {} + public void commitDaughterRegion(HRegionInfo hri) {} + public void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {} + public void commitMergedRegion(HRegionInfo hri) {} + public void createSplitsDir() throws IOException {} + public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f, + final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy) + throws IOException { return null; } + public Path getMergesDir() { return null; } + public Path getMergesDir(final HRegionInfo hri) { return null; } + public void createMergesDir() throws IOException {} + public void assertReferenceFileCountOfSplitsDir(int expectedReferenceFileCount, HRegionInfo daughter) {} + public void assertReferenceFileCountOfDaughterDir(int expectedReferenceFileCount, HRegionInfo daughter) {} + public void cleanupSplitsDir() throws IOException {} + public void cleanupMergesDir() throws IOException {} + public Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName, + final StoreFile f, final Path mergedDir) + throws IOException { return null; } + + public boolean hasReferences(final String familyName) { return false; } + public boolean hasReferences(final HTableDescriptor htd) { return false; } + + public void openFamily(final String family) throws IOException {} + + // ========================================================================== + // MAYBE methods + // ========================================================================== + public abstract HRegionInfo getRegionInfo(); + + public TableName getTable() { + return getRegionInfo().getTable(); + } + + // ========================================================================== + // PUBLIC methods - create/open/destroy + // ========================================================================== + public abstract void open(boolean rdonly) throws IOException; + public abstract void create() throws IOException; + public abstract void destroy() throws IOException; + + // ========================================================================== + // PUBLIC Static/Factory methods - create/open/destroy + // ========================================================================== + public static HRegionFileSystem createSnapshot(final Configuration conf, + final HRegionInfo regionInfo) throws IOException { + // we need a dir on open, create if we want to reuse... + return null; + } + + public static HRegionFileSystem create(final Configuration conf, + final HRegionInfo regionInfo) throws IOException { + HRegionFileSystem rfs = getInstance(conf, regionInfo); + rfs.create(); + return rfs; + } + + public static HRegionFileSystem destroy(final Configuration conf, + final HRegionInfo regionInfo) throws IOException { + HRegionFileSystem rfs = getInstance(conf, regionInfo); + rfs.destroy(); + return rfs; + } + + public static HRegionFileSystem open(final Configuration conf, + final HRegionInfo regionInfo, boolean readOnly) throws IOException { + HRegionFileSystem rfs = getInstance(conf, regionInfo); + rfs.open(readOnly); + return rfs; + } + + private static HRegionFileSystem getInstance(final Configuration conf, + final HRegionInfo regionInfo) throws IOException { + String fsType = conf.get("hbase.fs.layout.type").toLowerCase(); + switch (fsType) { + case "legacy": + return new LegacyRegionFileSystem(conf, regionInfo); + case "hierarchical": + return new HierarchicalRegionFileSystem(conf, regionInfo); + default: + throw new IOException("Invalid filesystem type " + fsType); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java new file mode 100644 index 0000000..fef4686 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java @@ -0,0 +1,49 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.FSUtils; + +public class HierarchicalMasterFileSystem extends LegacyMasterFileSystem { + public HierarchicalMasterFileSystem(Configuration conf, Path rootDir) { + super(conf, rootDir); + } + + public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException { + /* + for (...ns bucket..) { + for (...rs...) { + ... + } + } + */ + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java new file mode 100644 index 0000000..d6a6a97 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java @@ -0,0 +1,88 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; + [email protected] +public class HierarchicalRegionFileSystem extends LegacyRegionFileSystem { + private static final Log LOG = LogFactory.getLog(HierarchicalRegionFileSystem.class); + + private static final String OLD_REGION_NAME_PADDING = "abcdef1234abcdef1234abcdef1234ab"; + + /** Number of characters for DIR name, 4 characters for 16^4 = 65536 buckets. */ + public static final int HUMONGOUS_DIR_NAME_SIZE = 4; + + public HierarchicalRegionFileSystem(Configuration conf, HRegionInfo hri) { + super(conf, hri); + } + + // ========================================================================== + // PROTECTED FS-Layout Internals + // ========================================================================== + @Override + protected Path getRegionDir(Path baseDir) { + return getHumongousRegionDir(baseDir, getTable(), getRegionInfo().getEncodedName()); + } + + private Path getHumongousRegionDir(Path baseDir, TableName tableName, String name) { + if (name.length() != HRegionInfo.MD5_HEX_LENGTH) { + String table = tableName.getQualifierAsString(); + String namespace = tableName.getNamespaceAsString(); + + // Meta and old root table use the old encoded name format still + if (!namespace.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { + throw new IllegalArgumentException("The region with encoded name " + name + + " is not a humongous region, cannot get humongous region dir from it."); + } + if (!table.equals(TableName.META_TABLE_NAME.getQualifierAsString()) && + !table.equals(TableName.OLD_ROOT_TABLE_NAME.getQualifierAsString())) { + throw new IllegalArgumentException("The region with encoded name " + name + + " is not a humongous region, cannot get humongous region dir from it."); + } + + // Add padding to guarantee we will have enough characters + return new Path(new Path(baseDir, makeBucketName(name, OLD_REGION_NAME_PADDING)), name); + } + return new Path(new Path(baseDir, makeBucketName(name, null)), name); + } + + private String makeBucketName(String regionName, String padding) { + if (padding != null) { + regionName = regionName + padding; + } + return regionName.substring(HRegionInfo.MD5_HEX_LENGTH - HUMONGOUS_DIR_NAME_SIZE); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java new file mode 100644 index 0000000..87ee61b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java @@ -0,0 +1,152 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; + [email protected] +public class LegacyMasterFileSystem extends HMasterFileSystem { + private final Path activeNsDir; + private final Path rootDir; + + private final FileSystem fs; + + public LegacyMasterFileSystem(Configuration conf, Path rootDir) { + this.fs = rootDir.getFileSystem(conf); + this.rootDir = rootDir; + this.activeNsDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR); + } + + public Collection<String> getNamespaces() throws IOException { + return getNamespaces(activeNsDir); + } + + protected Collection<String> getNamespaces(Path baseDir) throws IOException { + FileStatus[] activeNsDirs = fs.globStatus(new Path(baseDir, "*")); + if (activeNsDirs == null || activeNsDirs.length == 0) { + return Collections.emptyList(); + } + ArrayList<String> namespaces = new ArrayList<String>(activeNsDirs.length); + for (int i = 0; i < activeNsDirs.length; ++i) { + namespaces.add(activeNsDirs[i].getPath().getName()); + } + return namespaces; + } + + public Collection<TableName> getTables(String namespace) throws IOException { + Path baseDir = new Path(activeNsDir, namespace); + FileStatus[] dirs = fs.listStatus(baseDir, new UserTableDirFilter(fs)); + if (dirs == null || dirs.length == 0) { + return Collections.emptyList(); + } + + ArrayList<TableName> tables = new ArrayList<TableName>(dirs.length); + for (int i = 0; i < dirs.length; ++i) { + tables.add(TableName.valueOf(namespace, dirs[i].getPath().getName())); + } + return tables; + } + + public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException { + return null; + } + + protected Collection<HRegionInfo> getRegions(Path baseDir) throws IOException { + FileStatus[] dirs = fs.listStatus(baseDir, new RegionDirFilter(fs)); + if (dirs == null || dirs.length == 0) { + return Collections.emptyList(); + } + + ArrayList<HRegionInfo> hriList = new ArrayList<HRegionInfo>(dirs.length); + for (int i = 0; i < dirs.length; ++i) { + // TODO: Load HRI + } + return hriList; + } + + /** + * A {@link PathFilter} that returns usertable directories. To get all directories use the + * {@link BlackListDirFilter} with a <tt>null</tt> blacklist + */ + private static class UserTableDirFilter extends FSUtils.BlackListDirFilter { + public UserTableDirFilter(FileSystem fs) { + super(fs, HConstants.HBASE_NON_TABLE_DIRS); + } + + protected boolean isValidName(final String name) { + if (!super.isValidName(name)) + return false; + + try { + TableName.isLegalTableQualifierName(Bytes.toBytes(name)); + } catch (IllegalArgumentException e) { + LOG.info("INVALID NAME " + name); + return false; + } + return true; + } + } + + /** + * Filter for all dirs that don't start with '.' + */ + private static class RegionDirFilter implements PathFilter { + // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names. + final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$"); + final FileSystem fs; + + public RegionDirFilter(FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path rd) { + if (!regionDirPattern.matcher(rd.getName()).matches()) { + return false; + } + + try { + return fs.getFileStatus(rd).isDirectory(); + } catch (IOException ioe) { + // Maybe the file was moved or the fs was disconnected. + LOG.warn("Skipping file " + rd +" due to IOException", ioe); + return false; + } + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java new file mode 100644 index 0000000..9f8cc70 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java @@ -0,0 +1,293 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSHDFSUtils; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; + +import com.google.common.annotations.VisibleForTesting; + [email protected] +public class LegacyRegionFileSystem extends HRegionFileSystem { + private static final Log LOG = LogFactory.getLog(LegacyRegionFileSystem.class); + + /** Temporary subdirectory of the region directory used for compaction output. */ + protected static final String REGION_TEMP_DIR = ".tmp"; + + private final Configuration conf; + private final HRegionInfo hri; + private final Path regionDir; + private final FileSystem fs; + + public LegacyRegionFileSystem(Configuration conf, HRegionInfo hri) { + this.fs = FSUtils.getCurrentFileSystem(conf); + this.hri = hri; + this.conf = conf; + this.regionDir = getRegionDir(getTableDir(getRootDir())); + } + + // ========================================================================== + // ??? methods - do we still need this stuff in 2.0? + // ========================================================================== + @Override + public void writeRecoveryCheckPoint() throws IOException { + //checkRegionInfoOnFilesystem + } + + @Override + public void cleanup() throws IOException { + /* + // Remove temporary data left over from old regions + status.setStatus("Cleaning up temporary data from old regions"); + fs.cleanupTempDir(); + + status.setStatus("Cleaning up detritus from prior splits"); + // Get rid of any splits or merges that were lost in-progress. Clean out + // these directories here on open. We may be opening a region that was + // being split but we crashed in the middle of it all. + fs.cleanupAnySplitDetritus(); + fs.cleanupMergesDir(); + */ + } + + // ========================================================================== + // PUBLIC methods - create/open/destroy + // ========================================================================== + @Override + public void open(boolean rdonly) throws IOException { + } + + @Override + public void create() throws IOException { + } + + @Override + public void destroy() throws IOException { + } + + // ========================================================================== + // PUBLIC methods - add/remove/list store files + // ========================================================================== + public Path createTempName(final String suffix) { + return new Path(getTempDir(), generateUniqueName(suffix)); + } + + public Path commitStoreFile(final String familyName, final Path buildPath, + final long seqNum, final boolean generateNewName) throws IOException { + return null; + } + + public void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) + throws IOException { + } + + public Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum) + throws IOException { + return null; + } + + public void removeStoreFile(String familyName, Path filePath) + throws IOException { + } + + public void removeStoreFiles(String familyName, Collection<StoreFile> storeFiles) + throws IOException { + } + + public Path getStoreFilePath(final String familyName, final String fileName) { + return null; + } + + /** + * Returns the store files available for the family. + * This methods performs the filtering based on the valid store files. + * @param familyName Column Family Name + * @return a set of {@link StoreFileInfo} for the specified family. + */ + public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate) + throws IOException { + /* + Path familyDir = getStoreDir(this.regionDir, familyName); + FileStatus[] files = FSUtils.listStatus(this.fs, familyDir); + if (files == null) { + LOG.debug("No StoreFiles for: " + familyDir); + return null; + } + + ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length); + for (FileStatus status: files) { + if (validate && !StoreFileInfo.isValid(status)) { + LOG.warn("Invalid StoreFile: " + status.getPath()); + continue; + } + StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, + regionInfoForFs, familyName, status.getPath()); + storeFiles.add(info); + } + return storeFiles; + */ + return null; + } + + public StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) + throws IOException { + return null; + } + + /** + * @return the set of families present on disk + * @throws IOException + */ + public Collection<String> getFamilies() throws IOException { + FileStatus[] fds = FSUtils.listStatus(fs, this.regionDir, new FamilyDirFilter(fs)); + if (fds == null) return null; + + ArrayList<String> families = new ArrayList<String>(fds.length); + for (FileStatus status: fds) { + families.add(status.getPath().getName()); + } + + return families; + } + + // ========================================================================== + // PROTECTED Internals + // ========================================================================== + + /** + * Generate a unique file name, used by createTempName() and commitStoreFile() + * @param suffix extra information to append to the generated name + * @return Unique file name + */ + protected static String generateUniqueName(final String suffix) { + String name = UUID.randomUUID().toString().replaceAll("-", ""); + if (suffix != null) name += suffix; + return name; + } + + public HRegionInfo getRegionInfo() { + return hri; + } + + // ========================================================================== + // PROTECTED FS-Layout Internals + // ========================================================================== + /** @return {@link Path} to the region's temp directory, used for file creations */ + Path getTempDir() { + return new Path(getRegionDir(), REGION_TEMP_DIR); + } + + protected Path getRootDir() { + return new Path("/"); + } + + protected Path getTableDir(Path baseDir) { + TableName table = getTable(); + return new Path(baseDir, new Path(table.getNamespaceAsString(), table.getQualifierAsString())); + } + + protected Path getRegionDir(Path baseDir) { + return new Path(baseDir, getRegionInfo().getEncodedName()); + } + + protected Path getStoreDir(Path baseDir, String familyName) { + return new Path(baseDir, familyName); + } + + protected Path getStoreFilePath(Path baseDir, String family, String name) { + return getStoreFilePath(getStoreDir(baseDir, family), name); + } + + protected Path getStoreFilePath(Path baseDir, String name) { + return new Path(baseDir, name); + } + + @Override + public String toString() { + return String.format("%s(%s)", getClass().getName(), regionDir); + } + + /** + * Filter for all dirs that are legal column family names. This is generally used for colfam + * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>. + */ + private static class FamilyDirFilter implements PathFilter { + final FileSystem fs; + + public FamilyDirFilter(FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path rd) { + try { + // throws IAE if invalid + HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName())); + } catch (IllegalArgumentException iae) { + // path name is an invalid family name and thus is excluded. + return false; + } + + try { + return fs.getFileStatus(rd).isDirectory(); + } catch (IOException ioe) { + // Maybe the file was moved or the fs was disconnected. + LOG.warn("Skipping file " + rd +" due to IOException", ioe); + return false; + } + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java deleted file mode 100644 index 8fd651f..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.fs.layout; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.FSUtils; - -public abstract class AFsLayout { - protected final static Log LOG = LogFactory.getLog(AFsLayout.class); - - public abstract HRegionFileSystemFactory getHRegionFileSystemFactory(); - - public abstract Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName); - - public abstract Path getRegionArchiveDir(Path rootDir, TableName tableName, Path regiondir); - - public abstract Path getTableDirFromRegionDir(Path regionDir); - - public abstract List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException; - - public abstract List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, FSUtils.RegionDirFilter filter) throws IOException; - - public abstract Path getRegionDir(Path tableDir, String name); - - protected AFsLayout() { - super(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java deleted file mode 100644 index 0eb2489..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.fs.layout; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.util.Pair; - -public abstract class AHFileLinkManager { - public abstract HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern) throws IOException; - - public abstract HFileLink buildFromHFileLinkPattern(final Path rootDir, final Path archiveDir, final Path hFileLinkPattern); - - public abstract Path createPath(final TableName table, final String region, final String family, final String hfile); - - public abstract HFileLink build(final Configuration conf, final TableName table, final String region, final String family, final String hfile) - throws IOException; - - public abstract boolean isHFileLink(final Path path); - - public abstract boolean isHFileLink(String fileName); - - public abstract String getReferencedHFileName(final String fileName); - - public abstract Path getHFileLinkPatternRelativePath(Path path); - - public abstract String getReferencedRegionName(final String fileName); - - public abstract TableName getReferencedTableName(final String fileName); - - public abstract String createHFileLinkName(final HRegionInfo hfileRegionInfo, final String hfileName); - - public abstract String createHFileLinkName(final TableName tableName, final String regionName, final String hfileName); - - public abstract boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, final String hfileName) - throws IOException; - - public abstract boolean createFromHFileLink(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final String hfileLinkName) throws IOException; - - public abstract String createBackReferenceName(final String tableNameStr, final String regionName); - - public abstract Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath); - - public abstract Pair<TableName, String> parseBackReferenceName(String name); - - public abstract Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath) throws IOException; - - public AHFileLinkManager() { - super(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java deleted file mode 100644 index c97d2a0..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.fs.layout; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import com.google.common.primitives.Ints; - -import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.Nullable; - -/** - * This class exists mostly to allow us to access the layouts statically for convenience, as - * though the layout classes are Util files. - */ -public class FsLayout { - private static final Log LOG = LogFactory.getLog(FsLayout.class); - - public static final String FS_LAYOUT_CHOICE = "hbase.fs.layout.choose"; - public static final String FS_LAYOUT_DETECT = "hbase.fs.layout.detect"; - public static final String FS_LAYOUT_DETECT_STRICT = "hbase.fs.layout.detect.strict"; - public static final String FS_LAYOUT_FILE_NAME = ".fslayout"; - - // TODO: How can we avoid having a volatile variable (slightly slower reads)? - // TODO: Move FsLayout class/contents into FSUtils? - private static volatile AFsLayout fsLayout = null; - - public static final PathFilter FS_LAYOUT_PATHFILTER = new PathFilter() { - @Override - public boolean accept(Path p) { - return p.getName().equals(FS_LAYOUT_FILE_NAME); - } - }; - - @VisibleForTesting - static AFsLayout getRaw() { - return fsLayout; - } - - public static AFsLayout get() { - AFsLayout curLayout = fsLayout; - if (curLayout == null) { - return initialize(null); - } else { - return curLayout; - } - } - - @VisibleForTesting - public static void reset() { - LOG.debug("Resetting FS layout to null"); - fsLayout = null; - } - - @VisibleForTesting - public static void setLayoutForTesting(@NonNull AFsLayout inputLayout) { - LOG.debug("Setting FS layout to: " + inputLayout.getClass().getSimpleName()); - fsLayout = inputLayout; - } - - /* - * TODO: Should this be required to be called manually? - * Maybe call it manually in some processes (master/regionserver) and automatically everywhere else - */ - @VisibleForTesting - static synchronized AFsLayout initialize(Configuration conf) { - try { - if (fsLayout != null) { - LOG.debug("Already initialized FS layout, not going to re-initialize"); - return fsLayout; - } - if (conf == null) { - conf = HBaseConfiguration.create(); - } - String choice = conf.get(FS_LAYOUT_CHOICE, null); - boolean autodetect = conf.getBoolean(FS_LAYOUT_DETECT, false); - if (choice != null && autodetect) { - throw new IllegalStateException("Configuration both chooses a layout and " - + "tries to automatically detect the layout"); - } - if (choice != null) { - Class<?> layoutClass = Class.forName(choice); - Method getMethod = layoutClass.getMethod("get"); - return (AFsLayout) getMethod.invoke(null); - } - if (autodetect) { - LOG.debug("Trying to detect hbase layout on filesystem"); - FileSystem fs = FSUtils.getCurrentFileSystem(conf); - Path rootDir = FSUtils.getRootDir(conf); - AFsLayout fsLayoutFromFile = readLayoutFile(fs, rootDir); - if (fsLayoutFromFile == null) { - if (conf.getBoolean(FS_LAYOUT_DETECT_STRICT, false)) { - throw new IllegalStateException("Tried to detect fs layout, but there was no layout file at the root!"); - } else { - LOG.debug("Didn't find a layout file, assuming classical hbase fs layout"); - fsLayout = StandardHBaseFsLayout.get(); - } - } else { - LOG.info("Detected hbase fs layout: " + fsLayoutFromFile.getClass().getSimpleName()); - fsLayout = fsLayoutFromFile; - } - } else { - fsLayout = StandardHBaseFsLayout.get(); - } - } catch (Exception e) { - Throwables.propagate(e); - } - return fsLayout; - } - - public static AFsLayout readLayoutFile(FileSystem fs, Path rootDir) - throws FileNotFoundException, IOException, ClassNotFoundException, - NoSuchMethodException, SecurityException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME); - FileStatus[] statuses = fs.listStatus(rootDir, FS_LAYOUT_PATHFILTER); - if (statuses.length != 1) { - return null; - } - FileStatus stat = statuses[0]; - int len = Ints.checkedCast(stat.getLen()); - byte[] inputStreamBytes = new byte[len]; - FSDataInputStream inputStream = fs.open(layoutFilePath); - inputStream.readFully(inputStreamBytes); - inputStream.close(); - String layoutClassName = Bytes.toString(inputStreamBytes); - Class<?> layoutClass = Class.forName(layoutClassName); - Method getMethod = layoutClass.getMethod("get"); - return (AFsLayout) getMethod.invoke(null); - } - - public static void writeLayoutFile(FileSystem fs, Path rootDir, AFsLayout fsLayout, boolean overwrite) - throws IOException { - Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME); - FSDataOutputStream outputStream = fs.create(layoutFilePath, overwrite); - try { - outputStream.write(Bytes.toBytes(fsLayout.getClass().getCanonicalName())); - } finally { - outputStream.close(); - } - } - - public static boolean deleteLayoutFile(FileSystem fs, Path rootDir) throws IOException { - Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME); - return fs.delete(layoutFilePath, false); - } - - public static HRegionFileSystemFactory getHRegionFileSystemFactory() { - return get().getHRegionFileSystemFactory(); - } - - public static Path getRegionDir(Path tableDir, HRegionInfo regionInfo) { - return getRegionDir(tableDir, regionInfo.getEncodedName()); - } - - public static Path getRegionDir(Path tableDir, String name) { - return get().getRegionDir(tableDir, name); - } - - @Nullable - public static List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter) throws IOException { - return get().getRegionDirFileStats(fs, tableDir, filter); - } - - public static List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException { - return get().getRegionDirPaths(fs, tableDir); - } - - public static Path getTableDirFromRegionDir(Path regionDir) { - return get().getTableDirFromRegionDir(regionDir); - } - - public static Path getRegionArchiveDir(Path rootDir, TableName tableName, Path regiondir) { - return get().getRegionArchiveDir(rootDir, tableName, regiondir); - } - - public static Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) { - return get().makeHFileLinkPath(snapshotManifest, regionInfo, familyName, hfileName); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java deleted file mode 100644 index d17428a..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.fs.layout; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -import org.apache.hadoop.hbase.regionserver.HierarchicalHRegionFileSystemFactory; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; - - -public class HierarchicalFsLayout extends AFsLayout { - private static final String OLD_REGION_NAME_PADDING = "abcdef1234abcdef1234abcdef1234ab"; - private static final HierarchicalFsLayout LAYOUT = new HierarchicalFsLayout(); - - static { - assert OLD_REGION_NAME_PADDING.length() == 32; - } - - public static HierarchicalFsLayout get() { return LAYOUT; } - - private HierarchicalFsLayout() { } - - @Override - public HierarchicalHRegionFileSystemFactory getHRegionFileSystemFactory() { - return new HierarchicalHRegionFileSystemFactory(); - } - - @Override - public Path getRegionDir(Path tableDir, String name) { - return getHumongousRegionDir(tableDir, name); - } - - private Path getHumongousRegionDir(final Path tabledir, final String name) { - if (name.length() != HRegionInfo.MD5_HEX_LENGTH) { - String table = tabledir.getName(); - String namespace = tabledir.getParent().getName(); - - // Meta and old root table use the old encoded name format still - if (!namespace.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { - throw new IllegalArgumentException("The region with encoded name " + name - + " is not a humongous region, cannot get humongous region dir from it."); - } - if (!table.equals(TableName.META_TABLE_NAME.getQualifierAsString()) && - !table.equals(TableName.OLD_ROOT_TABLE_NAME.getQualifierAsString())) { - throw new IllegalArgumentException("The region with encoded name " + name - + " is not a humongous region, cannot get humongous region dir from it."); - } - - // Add padding to guarantee we will have enough characters - return new Path(new Path(tabledir, makeBucketName(name, OLD_REGION_NAME_PADDING)), name); - } - return new Path(new Path(tabledir, makeBucketName(name, null)), name); - } - - private String makeBucketName(String regionName, String padding) { - if (padding != null) { - regionName = regionName + padding; - } - return regionName.substring(HRegionInfo.MD5_HEX_LENGTH - - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE); - } - - @Override - public List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter) - throws IOException { - FileStatus[] buckets = FSUtils.listStatus(fs, tableDir); - if (buckets == null) { - return null; - } - List<FileStatus> stats = new ArrayList<FileStatus>(); - for (FileStatus bucket : buckets) { - FileStatus[] regionDirs = null; - if (filter != null) { - regionDirs = fs.listStatus(bucket.getPath(), filter); - } else { - regionDirs = fs.listStatus(bucket.getPath()); - } - for (FileStatus regionDir : regionDirs) { - stats.add(regionDir); - } - } - if (stats.size() == 0) { - return null; - } - return stats; - } - - /** - * Given a particular table dir, return all the regiondirs inside it, excluding files such as - * .tableinfo - * @param fs A file system for the Path - * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir> - * @return List of paths to valid region directories in table dir. - * @throws IOException - */ - @Override - public List<Path> getRegionDirPaths(final FileSystem fs, final Path tableDir) throws IOException { - // assumes we are in a table dir. - FileStatus[] rds = fs.listStatus(tableDir, new FSUtils.RegionDirFilter(fs)); - List<Path> regionDirs = new ArrayList<Path>(); - for (FileStatus rdfs : rds) { - // get all region dirs from bucket dir - FileStatus[] bucket_rds = fs.listStatus(rdfs.getPath(), - new FSUtils.RegionDirFilter(fs)); - for (FileStatus bucket_rdfs : bucket_rds) { - regionDirs.add(bucket_rdfs.getPath()); - } - } - return regionDirs; - } - - @Override - public Path getTableDirFromRegionDir(Path regionDir) { - return regionDir.getParent().getParent(); - } - - /** - * Get the archive directory for a given region under the specified table - * @param tableName the table name. Cannot be null. - * @param regiondir the path to the region directory. Cannot be null. - * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it - * should not be archived - */ - @Override - public Path getRegionArchiveDir(Path rootDir, - TableName tableName, - Path regiondir) { - // get the archive directory for a table - Path archiveDir = HFileArchiveUtil.getTableArchivePath(rootDir, tableName); - - // then add on the region path under the archive - String encodedRegionName = regiondir.getName(); - String parentName = regiondir.getParent().getName(); - - return new Path(archiveDir, new Path(parentName, encodedRegionName)); - } - - @Override - public Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) { - return new Path(new Path(getHumongousRegionDir(snapshotManifest.getSnapshotDir(), - regionInfo.getEncodedName()), familyName), hfileName); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java deleted file mode 100644 index 98edb78..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.fs.layout; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; - - -public class StandardHBaseFsLayout extends AFsLayout { - private static final StandardHBaseFsLayout LAYOUT = new StandardHBaseFsLayout(); - - public static StandardHBaseFsLayout get() { return LAYOUT; } - - private StandardHBaseFsLayout() { } - - @Override - public HRegionFileSystemFactory getHRegionFileSystemFactory() { - return new HRegionFileSystemFactory(); - } - - @Override - public Path getRegionDir(Path tableDir, String name) { - return new Path(tableDir, name); - } - - @Override - public List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter) - throws IOException { - FileStatus[] rds = FSUtils.listStatus(fs, tableDir, filter); - if (rds == null) { - return null; - } - List<FileStatus> regionStatus = new ArrayList<FileStatus>(rds.length); - for (FileStatus rdfs : rds) { - regionStatus.add(rdfs); - } - return regionStatus; - } - - /** - * Given a particular table dir, return all the regiondirs inside it, excluding files such as - * .tableinfo - * @param fs A file system for the Path - * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir> - * @return List of paths to valid region directories in table dir. - * @throws IOException - */ - @Override - public List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException { - // assumes we are in a table dir. - FileStatus[] rds = fs.listStatus(tableDir, new FSUtils.RegionDirFilter(fs)); - List<Path> regionDirs = new ArrayList<Path>(); - for (FileStatus rdfs : rds) { - regionDirs.add(rdfs.getPath()); - } - return regionDirs; - } - - @Override - public Path getTableDirFromRegionDir(Path regionDir) { - return regionDir.getParent(); - } - - /** - * Get the archive directory for a given region under the specified table - * @param tableName the table name. Cannot be null. - * @param regiondir the path to the region directory. Cannot be null. - * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it - * should not be archived - */ - @Override - public Path getRegionArchiveDir(Path rootDir, - TableName tableName, - Path regiondir) { - // get the archive directory for a table - Path archiveDir = HFileArchiveUtil.getTableArchivePath(rootDir, tableName); - // then add on the region path under the archive - String encodedRegionName = regiondir.getName(); - return new Path(archiveDir, encodedRegionName); - } - - @Override - public Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) { - return new Path(new Path(new Path(snapshotManifest.getSnapshotDir(), - regionInfo.getEncodedName()), familyName), hfileName); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index d18dada..4ad310d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -248,7 +248,7 @@ public class HFile { protected FileSystem fs; protected Path path; protected FSDataOutputStream ostream; - protected CellComparator comparator = + protected CellComparator comparator = CellComparator.COMPARATOR; protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; @@ -830,33 +830,6 @@ public class HFile { } /** - * Returns all HFiles belonging to the given region directory. Could return an - * empty list. - * - * @param fs The file system reference. - * @param regionDir The region directory to scan. - * @return The list of files found. - * @throws IOException When scanning the files fails. - */ - static List<Path> getStoreFiles(FileSystem fs, Path regionDir) - throws IOException { - List<Path> regionHFiles = new ArrayList<Path>(); - PathFilter dirFilter = new FSUtils.DirFilter(fs); - FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter); - for(FileStatus dir : familyDirs) { - FileStatus[] files = fs.listStatus(dir.getPath()); - for (FileStatus file : files) { - if (!file.isDirectory() && - (!file.getPath().toString().contains(HConstants.HREGION_OLDLOGDIR_NAME)) && - (!file.getPath().toString().contains(HConstants.RECOVERED_EDITS_DIR))) { - regionHFiles.add(file.getPath()); - } - } - } - return regionHFiles; - } - - /** * Checks the given {@link HFile} format version, and throws an exception if * invalid. Note that if the version number comes from an input file and has * not been verified, the caller needs to re-throw an {@link IOException} to http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java index 757ad1f..0067de6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java @@ -24,6 +24,7 @@ import java.io.DataInput; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; @@ -56,10 +57,11 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterUtil; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -181,31 +183,28 @@ public class HFilePrettyPrinter extends Configured implements Tool { String regionName = cmd.getOptionValue("r"); byte[] rn = Bytes.toBytes(regionName); HRegionInfo hri = HRegionInfo.parseRegionInfoFromRegionName(rn); - Path rootDir = FSUtils.getRootDir(getConf()); - Path tableDir = FSUtils.getTableDir(rootDir, TableName.valueOf( - hri.getTable().getNameAsString())); - FileSystem fs = FileSystem.get(getConf()); - Path regionDir = HRegionFileSystem.create(getConf(), fs, tableDir, hri).getRegionDir(); - if (verbose) - System.out.println("region dir -> " + regionDir); - List<Path> regionFiles = HFile.getStoreFiles(fs, - regionDir); - if (verbose) - System.out.println("Number of region files found -> " - + regionFiles.size()); - if (verbose) { + HRegionFileSystem rfs = HRegionFileSystem.open(getConf(), hri, true); + printIfVerbose("region dir -> " + rfs); + for (String family: rfs.getFamilies()) { + Collection<StoreFileInfo> storeFiles = rfs.getStoreFiles(family); + printIfVerbose("Number of region files found -> " + storeFiles.size()); int i = 1; - for (Path p : regionFiles) { - if (verbose) - System.out.println("Found file[" + i++ + "] -> " + p); + for (StoreFileInfo storeFile : storeFiles) { + printIfVerbose("Found file[%d] -> %s", i++, storeFile.getPath()); + files.add(storeFile.getPath()); } } - files.addAll(regionFiles); } return true; } + private void printIfVerbose(String format, Object... args) { + if (verbose) { + System.out.println(String.format(format, args)); + } + } + /** * Runs the command-line pretty-printer, and returns the desired command * exit code (zero for success, non-zero for failure). @@ -245,8 +244,7 @@ public class HFilePrettyPrinter extends Configured implements Tool { } private void processFile(Path file) throws IOException { - if (verbose) - System.out.println("Scanning -> " + file); + printIfVerbose("Scanning -> " + file); FileSystem fs = file.getFileSystem(getConf()); if (!fs.exists(file)) { System.err.println("ERROR, file doesnt exist: " + file); http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index 9d78c1b..86bacb5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.fs.layout.FsLayout; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.RegionDoesNotExistException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -188,8 +189,7 @@ public class CatalogJanitor extends ScheduledChore { HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable()); HRegionFileSystem regionFs = null; try { - regionFs = HRegionFileSystem.openRegionFromFileSystem( - this.services.getConfiguration(), fs, tabledir, mergedRegion, true); + regionFs = HRegionFileSystem.open(this.services.getConfiguration(), mergedRegion, true); } catch (IOException e) { LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName()); } @@ -368,25 +368,14 @@ public class CatalogJanitor extends ScheduledChore { Path rootdir = this.services.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable()); - HRegionFileSystem hrfs = HRegionFileSystem.create( - this.services.getConfiguration(), fs, tabledir, daughter); + HRegionFileSystem hrfs = HRegionFileSystem.create(this.services.getConfiguration(), daughter); Path daughterRegionDir = hrfs.getRegionDir(); - HRegionFileSystem regionFs = null; - - try { - if (!FSUtils.isExists(fs, daughterRegionDir)) { - return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); - } - } catch (IOException ioe) { - LOG.warn("Error trying to determine if daughter region exists, " + - "assuming exists and has references", ioe); - return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE); - } - + HRegionFileSystem regionFs; try { - regionFs = HRegionFileSystem.openRegionFromFileSystem( - this.services.getConfiguration(), fs, tabledir, daughter, true); + regionFs = HRegionFileSystem.open(this.services.getConfiguration(), daughter, true); + } catch (RegionDoesNotExistException e) { + return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); } catch (IOException e) { LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName() + ", to: " + parent.getEncodedName() + " assuming has references", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 24f3071..408199d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -51,10 +51,9 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.fs.HFileSystem; -import org.apache.hadoop.hbase.fs.layout.FsLayout; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.wal.DefaultWALProvider; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.util.Bytes; @@ -504,7 +503,7 @@ public class MasterFileSystem { return rd; } - + /** * Checks if meta region exists * @@ -625,7 +624,7 @@ public class MasterFileSystem { Path regionDir = FsLayout.getRegionDir(tableDir, region); Path familyDir = new Path(regionDir, Bytes.toString(familyName)); - + if (fs.delete(familyDir, true) == false) { if (fs.exists(familyDir)) { throw new IOException("Could not delete family " @@ -702,38 +701,7 @@ public class MasterFileSystem { this.services.getTableDescriptors().add(htd); return htd; } - - // TODO: Can't get rid of this in totality because the caller of this method - // (TestSplitTransactionOnCluster.testSSHCleanupDaughterRegionsOfABortedSplit) - // is testing something where the FS does not agree with the meta - // At best can just make an assertion about # of region dirs in MFS and not expose - // what they actually are - public List<Path> getRegionDirs(TableName tableName) throws IOException { - FileSystem fs = getFileSystem(); - Path rootDir = FSUtils.getRootDir(conf); - Path tableDir = FSUtils.getTableDir(rootDir, tableName); - return FsLayout.getRegionDirPaths(fs, tableDir); - } - - // Only returns region filesystems for regions in meta - // Will ignore anything on filesystem - public List<HRegionFileSystem> getRegionFileSystems(Configuration conf, - Connection connection, TableName tableName) throws IOException { - - FileSystem fs = getFileSystem(); - Path rootDir = FSUtils.getRootDir(conf); - Path tableDir = FSUtils.getTableDir(rootDir, tableName); - - List<HRegionInfo> regionInfos = MetaTableAccessor.getTableRegions(connection, tableName); - - List<HRegionFileSystem> results = new ArrayList<HRegionFileSystem>(); - for (HRegionInfo regionInfo : regionInfos) { - HRegionFileSystem hrfs = HRegionFileSystem.create(conf, fs, tableDir, regionInfo); - results.add(hrfs); - } - return results; - } - + /** * The function is used in SSH to set recovery mode based on configuration after all outstanding * log split tasks drained. http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index bd672dc..ded09bb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -46,7 +46,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -646,7 +646,7 @@ public class RegionStates { if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) == null) { regionOffline(hri); - HRegionFileSystem.deleteRegionDir(server.getConfiguration(), hri); + HRegionFileSystem.destroy(server.getConfiguration(), hri); } } catch (IOException e) { LOG.warn("Got exception while deleting " + hri + " directories from file system.", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java index 9d54b00..06905b9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java @@ -54,8 +54,8 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.fs.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController; import org.apache.hadoop.hbase.mapreduce.JobUtil;
