This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new e50ec76 HBASE-22961 Deprecate hbck1 in core
e50ec76 is described below
commit e50ec76ac4379988acddf05be1e89a64a16dca8f
Author: stack <[email protected]>
AuthorDate: Sat Aug 31 09:24:33 2019 -0700
HBASE-22961 Deprecate hbck1 in core
Adds deprecations on HBaseFsck and on supporting classes such as
the reporting Interface. Provides alternatives in FSUtils for
progress reporting and deprecates methods that use hbck1 facility.
Signed-off-by: Peter Somogyi <[email protected]>
---
.../java/org/apache/hadoop/hbase/util/FSUtils.java | 120 ++++++++++++++++++---
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +-
.../hadoop/hbase/util/HbckErrorReporter.java | 11 +-
3 files changed, 115 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 3af9713..f65f980 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1180,6 +1180,17 @@ public abstract class FSUtils extends CommonFSUtils {
}
/**
+ * Called every so-often by storefile map builder getTableStoreFilePathMap to
+ * report progress.
+ */
+ interface ProgressReporter {
+ /**
+ * @param status File or directory we are about to process.
+ */
+ void progress(FileStatus status);
+ }
+
+ /**
* Runs through the HBase rootdir/tablename and creates a reverse lookup map
for
* table StoreFile names to the full Path.
* <br>
@@ -1198,7 +1209,8 @@ public abstract class FSUtils extends CommonFSUtils {
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path>
map,
final FileSystem fs, final Path hbaseRootDir, TableName tableName)
throws IOException, InterruptedException {
- return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null,
null, null);
+ return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null,
null,
+ (ProgressReporter)null);
}
/**
@@ -1218,15 +1230,55 @@ public abstract class FSUtils extends CommonFSUtils {
* @param tableName name of the table to scan.
* @param sfFilter optional path filter to apply to store files
* @param executor optional executor service to parallelize this operation
- * @param errors ErrorReporter instance or null
+ * @param progressReporter Instance or null; gets called every time we move
to new region of
+ * family dir and for each store file.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
- * @throws InterruptedException
+ * @deprecated Since 2.3.0. For removal in hbase4. Use ProgressReporter
override instead.
*/
+ @Deprecated
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path>
resultMap,
final FileSystem fs, final Path hbaseRootDir, TableName tableName, final
PathFilter sfFilter,
- ExecutorService executor, final HbckErrorReporter errors)
+ ExecutorService executor, final HbckErrorReporter progressReporter)
throws IOException, InterruptedException {
+ return getTableStoreFilePathMap(resultMap, fs, hbaseRootDir, tableName,
sfFilter, executor,
+ new ProgressReporter() {
+ @Override
+ public void progress(FileStatus status) {
+ // status is not used in this implementation.
+ progressReporter.progress();
+ }
+ });
+ }
+
+ /*
+ * Runs through the HBase rootdir/tablename and creates a reverse lookup map
for
+ * table StoreFile names to the full Path. Note that because this method
can be called
+ * on a 'live' HBase system that we will skip files that no longer exist by
the time
+ * we traverse them and similarly the user of the result needs to consider
that some
+ * entries in this map may not exist by the time this call completes.
+ * <br>
+ * Example...<br>
+ * Key = 3944417774205889744 <br>
+ * Value =
hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ *
+ * @param resultMap map to add values. If null, this method will create and
populate one
+ * to return
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @param tableName name of the table to scan.
+ * @param sfFilter optional path filter to apply to store files
+ * @param executor optional executor service to parallelize this operation
+ * @param progressReporter Instance or null; gets called every time we move
to new region of
+ * family dir and for each store file.
+ * @return Map keyed by StoreFile name with a value of the full Path.
+ * @throws IOException When scanning the directory fails.
+ * @throws InterruptedException
+ */
+ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path>
resultMap,
+ final FileSystem fs, final Path hbaseRootDir, TableName tableName, final
PathFilter sfFilter,
+ ExecutorService executor, final ProgressReporter progressReporter)
+ throws IOException, InterruptedException {
final Map<String, Path> finalResultMap =
resultMap == null ? new ConcurrentHashMap<>(128, 0.75f, 32) :
resultMap;
@@ -1247,8 +1299,8 @@ public abstract class FSUtils extends CommonFSUtils {
final List<Future<?>> futures = new ArrayList<>(regionDirs.size());
for (FileStatus regionDir : regionDirs) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(regionDir);
}
final Path dd = regionDir.getPath();
@@ -1271,8 +1323,8 @@ public abstract class FSUtils extends CommonFSUtils {
return;
}
for (FileStatus familyDir : familyDirs) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(familyDir);
}
Path family = familyDir.getPath();
if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
@@ -1282,8 +1334,8 @@ public abstract class FSUtils extends CommonFSUtils {
// put in map
FileStatus[] familyStatus = fs.listStatus(family);
for (FileStatus sfStatus : familyStatus) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(sfStatus);
}
Path sf = sfStatus.getPath();
if (sfFilter == null || sfFilter.accept(sf)) {
@@ -1362,12 +1414,45 @@ public abstract class FSUtils extends CommonFSUtils {
* @param hbaseRootDir The root directory to scan.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
- * @throws InterruptedException
*/
- public static Map<String, Path> getTableStoreFilePathMap(
- final FileSystem fs, final Path hbaseRootDir)
+ public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
+ final Path hbaseRootDir)
throws IOException, InterruptedException {
- return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, null);
+ return getTableStoreFilePathMap(fs, hbaseRootDir, null, null,
(ProgressReporter)null);
+ }
+
+ /**
+ * Runs through the HBase rootdir and creates a reverse lookup map for
+ * table StoreFile names to the full Path.
+ * <br>
+ * Example...<br>
+ * Key = 3944417774205889744 <br>
+ * Value =
hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ *
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @param sfFilter optional path filter to apply to store files
+ * @param executor optional executor service to parallelize this operation
+ * @param progressReporter Instance or null; gets called every time we move
to new region of
+ * family dir and for each store file.
+ * @return Map keyed by StoreFile name with a value of the full Path.
+ * @throws IOException When scanning the directory fails.
+ * @deprecated Since 2.3.0. Will be removed in hbase4. Used {@link
+ * #getTableStoreFilePathMap(FileSystem, Path, PathFilter,
ExecutorService, ProgressReporter)}
+ */
+ @Deprecated
+ public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
+ final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,
+ HbckErrorReporter progressReporter)
+ throws IOException, InterruptedException {
+ return getTableStoreFilePathMap(fs, hbaseRootDir, sfFilter, executor,
+ new ProgressReporter() {
+ @Override
+ public void progress(FileStatus status) {
+ // status is not used in this implementation.
+ progressReporter.progress();
+ }
+ });
}
/**
@@ -1382,14 +1467,15 @@ public abstract class FSUtils extends CommonFSUtils {
* @param hbaseRootDir The root directory to scan.
* @param sfFilter optional path filter to apply to store files
* @param executor optional executor service to parallelize this operation
- * @param errors ErrorReporter instance or null
+ * @param progressReporter Instance or null; gets called every time we move
to new region of
+ * family dir and for each store file.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
* @throws InterruptedException
*/
public static Map<String, Path> getTableStoreFilePathMap(
final FileSystem fs, final Path hbaseRootDir, PathFilter sfFilter,
- ExecutorService executor, HbckErrorReporter errors)
+ ExecutorService executor, ProgressReporter progressReporter)
throws IOException, InterruptedException {
ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<>(1024, 0.75f,
32);
@@ -1399,7 +1485,7 @@ public abstract class FSUtils extends CommonFSUtils {
// only include the directory paths to tables
for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
getTableStoreFilePathMap(map, fs, hbaseRootDir,
- FSUtils.getTableName(tableDir), sfFilter, executor, errors);
+ FSUtils.getTableName(tableDir), sfFilter, executor,
progressReporter);
}
return map;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 35396bd..f549d49 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -150,7 +150,9 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Sets;
* HBaseFsck (hbck) is a tool for checking and repairing region consistency and
* table integrity problems in a corrupted HBase. This tool was written for
hbase-1.x. It does not
* work with hbase-2.x; it can read state but is not allowed to change state;
i.e. effect 'repair'.
- * See hbck2 (HBASE-19121) for a hbck tool for hbase2.
+ * Even though it can 'read' state, given how so much has changed in how
hbase1 and hbase2 operate,
+ * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2.
This class is
+ * deprecated.
*
* <p>
* Region consistency checks verify that hbase:meta, region deployment on
region
@@ -193,7 +195,9 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Sets;
* If hbck is run from the command line, there are a handful of arguments that
* can be used to limit the kinds of repairs hbck will do. See the code in
* {@link #printUsageAndExit()} for more details.
+ * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.
*/
+@Deprecated
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@InterfaceStability.Evolving
public class HBaseFsck extends Configured implements Closeable {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
index 500b690..52012df 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,12 +20,15 @@ package org.apache.hadoop.hbase.util;
import java.util.ArrayList;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
+/**
+ * Used by {@link HBaseFsck} reporting system.
+ * @deprecated Since 2.3.0. To be removed in hbase4. Use HBCK2 instead. Remove
when
+ * {@link HBaseFsck} is removed.
+ */
+@Deprecated
@InterfaceAudience.Private
[email protected]
public interface HbckErrorReporter {
-
enum ERROR_CODE {
UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE,
NOT_IN_META_HDFS, NOT_IN_META,
NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS,
SERVER_DOES_NOT_MATCH_META,
