This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new 63938dc HBASE-23371 [HBCK2] Provide client side method for removing
ghost regions in meta. (#45)
63938dc is described below
commit 63938dc918bc9a7adbb83d3f2ed384c30de72c77
Author: Wellington Ramos Chevreuil <[email protected]>
AuthorDate: Mon Jan 27 11:16:51 2020 +0000
HBASE-23371 [HBCK2] Provide client side method for removing ghost regions
in meta. (#45)
Signed-off-by: Josh Elser [email protected]
---
hbase-hbck2/README.md | 35 +++
.../org/apache/hbase/FsRegionsMetaRecoverer.java | 222 +++++++++++++++----
.../src/main/java/org/apache/hbase/HBCK2.java | 236 +++++++++++++--------
.../apache/hbase/TestFsRegionsMetaRecoverer.java | 107 ++++++++--
.../src/test/java/org/apache/hbase/TestHBCK2.java | 227 +++++++++++++++++++-
5 files changed, 669 insertions(+), 158 deletions(-)
diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index 869e0ac..6e6d9ea 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -148,6 +148,29 @@ Command:
to finish parent and children. This is SLOW, and dangerous so use
selectively. Does not always work.
+ extraRegionsInMeta <NAMESPACE|NAMESPACE:TABLENAME>...
+ Options:
+ -f, --fix fix meta by removing all extra regions found.
+ Reports regions present on hbase:meta, but with no related
+ directories on the file system. Needs hbase:meta to be online.
+ For each table name passed as parameter, performs diff
+ between regions available in hbase:meta and region dirs on the given
+ file system. Extra regions would get deleted from Meta
+ if passed the --fix option.
+ NOTE: Before deciding on use the "--fix" option, it's worth check if
+ reported extra regions are overlapping with existing valid regions.
+ If so, then "extraRegionsInMeta --fix" is indeed the optimal solution.
+ Otherwise, "assigns" command is the simpler solution, as it recreates
+ regions dirs in the filesystem, if not existing.
+ An example triggering extra regions report for tables 'table_1'
+ and 'table_2', under default namespace:
+ $ HBCK2 extraRegionsInMeta default:table_1 default:table_2
+ An example triggering extra regions report for table 'table_1'
+ under default namespace, and for all tables from namespace 'ns1':
+ $ HBCK2 extraRegionsInMeta default:table_1 ns1
+ Returns list of extra regions for each table passed as parameter, or
+ for each table on namespaces specified as parameter.
+
filesystem [OPTIONS] [<TABLENAME>...]
Options:
-f, --fix sideline corrupt hfiles, bad links, and references.
@@ -528,6 +551,18 @@ using the _addFsRegionsMissingInMeta_ command in _HBCK2_.
This command is less d
hbase than a full hbase:meta rebuild covered later, and it can be used even for
recovering the _namespace_ table region.
+### Extra Regions in hbase:meta region/table restore/rebuild
+
+There can also be situations where table regions have been removed file
system, but still
+have related entries on hbase:meta table. This may happen due to problems on
splitting, manual
+operation mistakes (like deleting/moving the region dir manually), or even
meta info data loss
+issues such as HBASE-21843.
+
+Such problem can be addressed with the Master online, using the
_removeExtraRegionsFromMeta_
+command in _HBCK2_. This command is less disruptive to hbase than a full
hbase:meta rebuild
+covered later. Also useful when this happens on versions that don't support
_fixMeta_ hbck2 option
+(any prior to "2.0.6", "2.1.6", "2.2.1", "2.3.0","3.0.0").
+
#### Online hbase:meta rebuild recipe
If hbase:meta corruption is not too critical, hbase would still be able to
bring it online. Even if namespace region
diff --git
a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
index c3dbbb8..e3cd643 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
@@ -25,6 +25,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +38,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -44,8 +50,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class implements the inner works required for check and recover
regions that wrongly
- * went missing in META.
+ * This class implements the inner works required for checking and recovering
regions that wrongly
+ * went missing in META, or are left present in META but with no equivalent FS
dir.
* Normally HBCK2 fix options rely on Master self-contained information to
recover/fix
* inconsistencies, but this an exception case where META table is in a broken
state.
* So, it assumes HDFS state as the source of truth, in other words, methods
provided here consider
@@ -70,7 +76,7 @@ public class FsRegionsMetaRecoverer implements Closeable {
this.fs = fileSystem;
}
- private List<Path> getTableRegionsDirs(String table) throws Exception {
+ private List<Path> getTableRegionsDirs(String table) throws IOException {
String hbaseRoot = this.config.get(HConstants.HBASE_DIR);
Path tableDir = FSUtils.getTableDir(new Path(hbaseRoot),
TableName.valueOf(table));
return FSUtils.getRegionDirs(fs, tableDir);
@@ -78,53 +84,183 @@ public class FsRegionsMetaRecoverer implements Closeable {
public Map<TableName,List<Path>> reportTablesMissingRegions(final
List<String> namespacesOrTables)
throws IOException {
- final Map<TableName,List<Path>> result = new HashMap<>();
- List<TableName> tableNames =
MetaTableAccessor.getTableStates(this.conn).keySet().stream()
- .filter(tableName -> {
- if(namespacesOrTables==null || namespacesOrTables.isEmpty()){
- return true;
- } else {
- Optional<String> findings = namespacesOrTables.stream().filter(
- name -> (name.indexOf(":") > 0) ?
- tableName.equals(TableName.valueOf(name)) :
- tableName.getNamespaceAsString().equals(name)).findFirst();
- return findings.isPresent();
- }
- }).collect(Collectors.toList());
- tableNames.stream().forEach(tableName -> {
- try {
- result.put(tableName,
-
findMissingRegionsInMETA(tableName.getNameWithNamespaceInclAsString()));
- } catch (Exception e) {
- LOG.warn("Can't get missing regions from meta", e);
- }
+ InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
+ return missingChecker.reportTablesRegions(namespacesOrTables,
this::findMissingRegionsInMETA);
+ }
+
+ public Map<TableName,List<RegionInfo>>
+ reportTablesExtraRegions(final List<String> namespacesOrTables) throws
IOException {
+ InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
+ return extraChecker.reportTablesRegions(namespacesOrTables,
this::findExtraRegionsInMETA);
+ }
+
+ List<Path> findMissingRegionsInMETA(String table) throws IOException {
+ InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
+ return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> {
+ ListUtils<Path, RegionInfo> utils = new ListUtils<>();
+ return utils.complement(dirs, regions, d -> d.getName(), r ->
r.getEncodedName());
});
- return result;
- }
-
- List<Path> findMissingRegionsInMETA(String table) throws Exception {
- final List<Path> missingRegions = new ArrayList<>();
- final List<Path> regionsDirs = getTableRegionsDirs(table);
- TableName tableName = TableName.valueOf(table);
- List<RegionInfo> regionInfos = MetaTableAccessor.
- getTableRegions(this.conn, tableName, false);
- HashSet<String> regionsInMeta = regionInfos.stream().map(info ->
- info.getEncodedName()).collect(Collectors.toCollection(HashSet::new));
- for(final Path regionDir : regionsDirs){
- if (!regionsInMeta.contains(regionDir.getName())) {
- LOG.debug(regionDir + "is not in META.");
- missingRegions.add(regionDir);
- }
- }
- return missingRegions;
}
- public void putRegionInfoFromHdfsInMeta(Path region) throws IOException {
+ List<RegionInfo> findExtraRegionsInMETA(String table) throws IOException {
+ InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
+ return extraChecker.checkRegionsInMETA(table, (regions,dirs) -> {
+ ListUtils<RegionInfo, Path> utils = new ListUtils<>();
+ return utils.complement(regions, dirs, r -> r.getEncodedName(), d ->
d.getName());
+ });
+ }
+
+ void putRegionInfoFromHdfsInMeta(Path region) throws IOException {
RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, region);
MetaTableAccessor.addRegionToMeta(conn, info);
}
- @Override public void close() throws IOException {
+ List<String> addMissingRegionsInMeta(List<Path> regionsPath) throws
IOException {
+ List<String> reAddedRegionsEncodedNames = new ArrayList<>();
+ for(Path regionPath : regionsPath){
+ this.putRegionInfoFromHdfsInMeta(regionPath);
+ reAddedRegionsEncodedNames.add(regionPath.getName());
+ }
+ return reAddedRegionsEncodedNames;
+ }
+
+ public List<Future<List<String>>> addMissingRegionsInMetaForTables(
+ List<String> nameSpaceOrTable) throws IOException {
+ InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
+ return
missingChecker.processRegionsMetaCleanup(this::reportTablesMissingRegions,
+ this::addMissingRegionsInMeta, nameSpaceOrTable);
+ }
+
+ public List<Future<List<String>>> removeExtraRegionsFromMetaForTables(
+ List<String> nameSpaceOrTable) throws IOException {
+ if(nameSpaceOrTable.size()>0) {
+ InternalMetaChecker<RegionInfo> extraChecker = new
InternalMetaChecker<>();
+ return
extraChecker.processRegionsMetaCleanup(this::reportTablesExtraRegions, regions
-> {
+ MetaTableAccessor.deleteRegionInfos(conn, regions);
+ return regions.stream().map(r ->
r.getEncodedName()).collect(Collectors.toList());
+ }, nameSpaceOrTable);
+ } else {
+ return null;
+ }
+ }
+
+
+ @Override
+ public void close() throws IOException {
this.conn.close();
}
+
+ private class InternalMetaChecker<T> {
+
+ List<T> checkRegionsInMETA(String table,
+ CheckingFunction<List<RegionInfo>, List<Path>, T> checkingFunction)
throws IOException {
+ final List<Path> regionsDirs = getTableRegionsDirs(table);
+ TableName tableName = TableName.valueOf(table);
+ List<RegionInfo> regions = MetaTableAccessor.
+ getTableRegions(FsRegionsMetaRecoverer.this.conn, tableName, false);
+ return checkingFunction.check(regions, regionsDirs);
+ }
+
+ Map<TableName,List<T>> reportTablesRegions(final List<String>
namespacesOrTables,
+ ExecFunction<List<T>, String> checkingFunction) throws IOException {
+ final Map<TableName,List<T>> result = new HashMap<>();
+ List<TableName> tableNames = MetaTableAccessor.
+ getTableStates(FsRegionsMetaRecoverer.this.conn).keySet().stream()
+ .filter(tableName -> {
+ if(namespacesOrTables==null || namespacesOrTables.isEmpty()){
+ return true;
+ } else {
+ Optional<String> findings = namespacesOrTables.stream().filter(
+ name -> (name.indexOf(":") > 0) ?
+ tableName.equals(TableName.valueOf(name)) :
+ tableName.getNamespaceAsString().equals(name)).findFirst();
+ return findings.isPresent();
+ }
+ }).collect(Collectors.toList());
+ tableNames.stream().forEach(tableName -> {
+ try {
+ result.put(tableName,
+
checkingFunction.execute(tableName.getNameWithNamespaceInclAsString()));
+ } catch (Exception e) {
+ LOG.warn("Can't get related regions report from meta", e);
+ }
+ });
+ return result;
+ }
+
+ List<Future<List<String>>> processRegionsMetaCleanup(
+ ExecFunction<Map<TableName, List<T>>, List<String>> reportFunction,
+ ExecFunction<List<String>, List<T>> execFunction,
+ List<String> nameSpaceOrTable) throws IOException {
+ ExecutorService executorService = Executors.newFixedThreadPool(
+ (nameSpaceOrTable == null ||
+ nameSpaceOrTable.size() >
Runtime.getRuntime().availableProcessors()) ?
+ Runtime.getRuntime().availableProcessors() :
+ nameSpaceOrTable.size());
+ List<Future<List<String>>> futures =
+ new ArrayList<>(nameSpaceOrTable == null ? 1 :
nameSpaceOrTable.size());
+ try {
+ try(final Admin admin = conn.getAdmin()) {
+ Map<TableName,List<T>> report =
reportFunction.execute(nameSpaceOrTable);
+ if(report.size() < 1) {
+ LOG.info("\nNo mismatches found in meta. Worth using related
reporting function " +
+ "first.\nYou are likely passing non-existent " +
+ "namespace or table. Note that table names should include the
namespace " +
+ "portion even for tables in the default namespace. " +
+ "See also the command usage.\n");
+ }
+ for (TableName tableName : report.keySet()) {
+ if(admin.tableExists(tableName)) {
+ futures.add(executorService.submit(new Callable<List<String>>() {
+ @Override
+ public List<String> call() throws Exception {
+ LOG.debug("running thread for {}",
tableName.getNameWithNamespaceInclAsString());
+ return execFunction.execute(report.get(tableName));
+ }
+ }));
+ } else {
+ LOG.warn("Table {} does not exist! Skipping...",
+ tableName.getNameWithNamespaceInclAsString());
+ }
+ }
+ boolean allDone;
+ do {
+ allDone = true;
+ for (Future<List<String>> f : futures) {
+ allDone &= f.isDone();
+ }
+ } while(!allDone);
+ }
+ } finally {
+ executorService.shutdown();
+ }
+ return futures;
+ }
+ }
+
+ @FunctionalInterface
+ interface CheckingFunction <RegionsList, DirList, T> {
+ List<T> check(RegionsList regions, DirList dirs) throws IOException;
+ }
+
+ @FunctionalInterface
+ interface ExecFunction<T, NamespaceOrTable> {
+ T execute(NamespaceOrTable name) throws IOException;
+ }
+
+ public class ListUtils<T1, T2> {
+ public List<T1> complement(List<T1> list1, List<T2> list2,
+ Function<T1, String> convertT1, Function<T2, String> convertT2) {
+ final List<T1> extraRegions = new ArrayList<>();
+ HashSet<String> baseSet = list2.stream().map(info ->
+ convertT2.apply(info)).collect(Collectors.toCollection(HashSet::new));
+ list1.forEach(region -> {
+ if(!baseSet.contains(convertT1.apply(region))) {
+ extraRegions.add(region);
+ }
+ });
+ return extraRegions;
+ }
+ }
+
}
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 0b1d33c..ead69c4 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -24,14 +24,13 @@ import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@@ -48,6 +47,7 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Hbck;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
@@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.master.RegionState;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
@@ -71,7 +70,6 @@ import
org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
-
/**
* HBase fixup tool version 2, for hbase-2.0.0+ clusters.
* Supercedes hbck1.
@@ -99,8 +97,8 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES =
"addFsRegionsMissingInMeta";
- private static final String ADD_MISSING_REGIONS_IN_META =
"addMissingRegionsInMeta";
private static final String REPORT_MISSING_REGIONS_IN_META =
"reportMissingRegionsInMeta";
+ static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
private Configuration conf;
static final String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0",
"3.0.0"};
private boolean skipCheck = false;
@@ -179,8 +177,8 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
Map<TableName,List<Path>> report;
try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
new FsRegionsMetaRecoverer(this.conf)) {
- List<String> names = nameSpaceOrTable != null ?
Arrays.asList(nameSpaceOrTable) : null;
- report = fsRegionsMetaRecoverer.reportTablesMissingRegions(names);
+ report = fsRegionsMetaRecoverer.reportTablesMissingRegions(
+ formatNameSpaceTableParam(nameSpaceOrTable));
} catch (IOException e) {
LOG.error("Error reporting missing regions: ", e);
throw e;
@@ -191,79 +189,72 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
return report;
}
- List<String> addMissingRegionsInMeta(List<Path> regionsPath) throws
IOException {
- List<String> reAddedRegionsEncodedNames = new ArrayList<>();
+ Map<TableName, List<String>> extraRegionsInMeta(String[] args)
+ throws Exception {
+ Options options = new Options();
+ Option fixOption = Option.builder("f").longOpt("fix").build();
+ options.addOption(fixOption);
+ // Parse command-line.
+ CommandLineParser parser = new DefaultParser();
+ CommandLine commandLine;
+ commandLine = parser.parse(options, args, false);
+ boolean fix = commandLine.hasOption(fixOption.getOpt());
+ Map<TableName, List<String>> result = new HashMap<>();
try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
- new FsRegionsMetaRecoverer(this.conf)) {
- for(Path regionPath : regionsPath){
- fsRegionsMetaRecoverer.putRegionInfoFromHdfsInMeta(regionPath);
- reAddedRegionsEncodedNames.add(regionPath.getName());
- }
- }
- return reAddedRegionsEncodedNames;
- }
-
- Pair<List<String>, List<ExecutionException>>
addMissingRegionsInMetaForTables(String...
- nameSpaceOrTable) throws IOException {
- ExecutorService executorService = Executors.newFixedThreadPool(
- (nameSpaceOrTable == null ||
- nameSpaceOrTable.length > Runtime.getRuntime().availableProcessors()) ?
- Runtime.getRuntime().availableProcessors() :
- nameSpaceOrTable.length);
- List<Future<List<String>>> futures =
- new ArrayList<>(nameSpaceOrTable == null ? 1 :
nameSpaceOrTable.length);
- final List<String> readdedRegionNames = new ArrayList<>();
- List<ExecutionException> executionErrors = new ArrayList<>();
- try {
- //reducing number of retries in case disable fails due to namespace
table region also missing
- this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
- try(ClusterConnection conn = connect();
- final Admin admin = conn.getAdmin()) {
- Map<TableName,List<Path>> report =
reportTablesWithMissingRegionsInMeta(nameSpaceOrTable);
- if(report.size() < 1) {
- LOG.info("\nNo missing regions in meta are found. Worth using " +
- "reportMissingRegionsInMeta first.\nYou are likely passing
non-existent " +
- "namespace or table. Note that table names should include
the namespace " +
- "portion even for tables in the default namespace. " +
- "See also the command usage.\n");
+ new FsRegionsMetaRecoverer(this.conf)) {
+ List<String> namespacesTables =
formatNameSpaceTableParam(commandLine.getArgs());
+ Map<TableName, List<RegionInfo>> reportMap =
+ fsRegionsMetaRecoverer.reportTablesExtraRegions(namespacesTables);
+ final List<String> toFix = new ArrayList<>();
+ reportMap.entrySet().forEach(e -> {
+ result.put(e.getKey(),
+
e.getValue().stream().map(r->r.getEncodedName()).collect(Collectors.toList()));
+ if(fix && e.getValue().size()>0){
+ toFix.add(e.getKey().getNameWithNamespaceInclAsString());
}
- for (TableName tableName : report.keySet()) {
- if(admin.tableExists(tableName)) {
- futures.add(executorService.submit(new Callable<List<String>>() {
- @Override
- public List<String> call() throws Exception {
- LOG.debug("running thread for {}",
tableName.getNameWithNamespaceInclAsString());
- return addMissingRegionsInMeta(report.get(tableName));
- }
- }));
- } else {
- LOG.warn("Table {} does not exist! Skipping...",
- tableName.getNameWithNamespaceInclAsString());
- }
- }
- for(Future<List<String>> f : futures){
- try {
- readdedRegionNames.addAll(f.get());
- } catch (ExecutionException e){
- //we want to allow potential running threads to finish, so we
collect execution
- //errors and show those later
- LOG.debug("Caught execution error: ", e);
- executionErrors.add(e);
+ });
+ if(fix) {
+ List<Future<List<String>>> removeResult =
+ fsRegionsMetaRecoverer.removeExtraRegionsFromMetaForTables(toFix);
+ if(removeResult!=null) {
+ int totalRegions = 0;
+ List<Exception> errors = new ArrayList<>();
+ for(Future<List<String>> f : removeResult){
+ try {
+ totalRegions += f.get().size();
+ } catch (ExecutionException|InterruptedException e){
+ errors.add(e);
+ }
}
+ System.out.println(formatRemovedRegionsMessage(totalRegions,
errors));
}
}
- } catch (IOException | InterruptedException e) {
- LOG.error("ERROR executing thread: ", e);
- throw new IOException(e);
- } finally {
- executorService.shutdown();
- }
- Pair<List<String>, List<ExecutionException>> result = new Pair<>();
- result.setFirst(readdedRegionNames);
- result.setSecond(executionErrors);
+ } catch (IOException e) {
+ LOG.error("Error on checking extra regions: ", e);
+ throw e;
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(formatExtraRegionsReport(result));
+ }
return result;
}
+ private List<String> formatNameSpaceTableParam(String... nameSpaceOrTable) {
+ return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null;
+ }
+
+ List<Future<List<String>>> addMissingRegionsInMetaForTables(String...
+ nameSpaceOrTable) throws IOException {
+ try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
+ new FsRegionsMetaRecoverer(this.conf)) {
+ return fsRegionsMetaRecoverer.addMissingRegionsInMetaForTables(
+ formatNameSpaceTableParam(nameSpaceOrTable));
+ } catch (IOException e) {
+ LOG.error("Error adding missing regions: ", e);
+ throw e;
+ }
+ }
+
List<Long> assigns(Hbck hbck, String [] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
@@ -374,6 +365,8 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
writer.println();
usageBypass(writer);
writer.println();
+ usageExtraRegionsInMeta(writer);
+ writer.println();
usageFilesystem(writer);
writer.println();
usageFixMeta(writer);
@@ -477,7 +470,7 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
writer.println(" noop. Otherwise, if 'HBCK Report' UI reports problems,
a run of");
writer.println(" " + FIX_META +
" will clear up hbase:meta issues. See 'HBase HBCK' UI");
- writer.println(" for how to generate new report.");
+ writer.println(" for how to generate new execute.");
writer.println(" SEE ALSO: " + REPORT_MISSING_REGIONS_IN_META);
}
@@ -490,6 +483,33 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
writer.println(" purge if '--fix'.");
}
+ private static void usageExtraRegionsInMeta(PrintWriter writer) {
+ writer.println(" " + EXTRA_REGIONS_IN_META + " <NAMESPACE|"
+ + "NAMESPACE:TABLENAME>...");
+ writer.println(" Options:");
+ writer.println(" -f, --fix fix meta by removing all extra regions
found.");
+ writer.println(" Reports regions present on hbase:meta, but with no
related ");
+ writer.println(" directories on the file system. Needs hbase:meta to be
online. ");
+ writer.println(" For each table name passed as parameter, performs
diff");
+ writer.println(" between regions available in hbase:meta and region dirs
on the given");
+ writer.println(" file system. Extra regions would get deleted from Meta
");
+ writer.println(" if passed the --fix option. ");
+ writer.println(" NOTE: Before deciding on use the \"--fix\" option, it's
worth check if");
+ writer.println(" reported extra regions are overlapping with existing
valid regions.");
+ writer.println(" If so, then \"extraRegionsInMeta --fix\" is indeed the
optimal solution. ");
+ writer.println(" Otherwise, \"assigns\" command is the simpler solution,
as it recreates ");
+ writer.println(" regions dirs in the filesystem, if not existing.");
+ writer.println(" An example triggering extra regions report for tables
'table_1'");
+ writer.println(" and 'table_2', under default namespace:");
+ writer.println(" $ HBCK2 " + EXTRA_REGIONS_IN_META +
+ " default:table_1 default:table_2");
+ writer.println(" An example triggering missing regions report for table
'table_1'");
+ writer.println(" under default namespace, and for all tables from
namespace 'ns1':");
+ writer.println(" $ HBCK2 " + EXTRA_REGIONS_IN_META + " default:table_1
ns1");
+ writer.println(" Returns list of extra regions for each table passed as
parameter, or");
+ writer.println(" for each table on namespaces specified as parameter.");
+ }
+
private static void usageReportMissingRegionsInMeta(PrintWriter writer) {
writer.println(" " + REPORT_MISSING_REGIONS_IN_META + " <NAMESPACE|"
+ "NAMESPACE:TABLENAME>...");
@@ -510,10 +530,10 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
writer.println(" It accepts a combination of multiple namespace and
tables. Table names");
writer.println(" should include the namespace portion, even for tables
in the default");
writer.println(" namespace, otherwise it will assume as a namespace
value.");
- writer.println(" An example triggering missing regions report for tables
'table_1'");
+ writer.println(" An example triggering missing regions execute for
tables 'table_1'");
writer.println(" and 'table_2', under default namespace:");
writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1
default:table_2");
- writer.println(" An example triggering missing regions report for table
'table_1'");
+ writer.println(" An example triggering missing regions execute for table
'table_1'");
writer.println(" under default namespace, and for all tables from
namespace 'ns1':");
writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1
ns1");
writer.println(" Returns list of missing regions for each table passed
as parameter, or");
@@ -816,9 +836,19 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
showErrorMessage(command + " takes one or more table names.");
return EXIT_FAILURE;
}
- Pair<List<String>, List<ExecutionException>> result =
+ List<Future<List<String>>> addedRegions =
addMissingRegionsInMetaForTables(purgeFirst(commands));
-
System.out.println(formatReAddedRegionsMessage(result.getFirst(),result.getSecond()));
+ List<String> regionNames = new ArrayList<>();
+ List<Exception> errors = new ArrayList<>();
+ for(Future<List<String>> f : addedRegions){
+ try {
+ regionNames.addAll(f.get());
+ } catch (InterruptedException | ExecutionException e) {
+ errors.add(e);
+ }
+ }
+ System.out.println(formatReAddedRegionsMessage(regionNames,
+ errors));
break;
case REPORT_MISSING_REGIONS_IN_META:
@@ -831,6 +861,16 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
break;
+ case EXTRA_REGIONS_IN_META:
+ try {
+ Map<TableName,List<String>> report =
+ extraRegionsInMeta(purgeFirst(commands));
+ System.out.println(formatExtraRegionsReport(report));
+ } catch (Exception e) {
+ return EXIT_FAILURE;
+ }
+ break;
+
default:
showErrorMessage("Unsupported command: " + command);
return EXIT_FAILURE;
@@ -843,22 +883,34 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
private String formatMissingRegionsInMetaReport(Map<TableName,List<Path>>
report) {
+ Function<Path,String> resolver = r -> r.getName();
+ String message = "Missing Regions for each table:\n\t";
+ return formatReportMessage(message, (HashMap)report, resolver);
+ }
+
+ private String formatExtraRegionsReport(Map<TableName,List<String>> report) {
+ String message = "Regions in Meta but having no equivalent dir, for each
table:\n\t";
+ return formatReportMessage(message, (HashMap)report, s -> s);
+ }
+
+ private String formatReportMessage(String reportMessage, Map<TableName,
List<?>> report,
+ Function resolver){
final StringBuilder builder = new StringBuilder();
if(report.size() < 1) {
- builder.append("\nNo reports are found. You are likely passing
non-existent " +
- "namespace or table. Note that table names should include the
namespace " +
- "portion even for tables in the default namespace. See also the
command usage.\n");
+ builder.append("\nNo reports were found. You are likely passing
non-existent " +
+ "namespace or table. Note that table names should include the
namespace " +
+ "portion even for tables in the default namespace. See also the
command usage.\n");
return builder.toString();
}
- builder.append("Missing Regions for each table:\n\t");
+ builder.append(reportMessage);
report.keySet().forEach(table -> {
builder.append(table);
if (!report.get(table).isEmpty()){
builder.append("->\n\t\t");
- report.get(table).forEach(region -> builder.append(region.getName())
+ report.get(table).forEach(region ->
builder.append(resolver.apply(region))
.append(" "));
} else {
- builder.append(" -> No missing regions");
+ builder.append(" -> No mismatching regions. This table is good!");
}
builder.append("\n\t");
});
@@ -866,7 +918,7 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
private String formatReAddedRegionsMessage(List<String> readdedRegionNames,
- List<ExecutionException> executionErrors) {
+ List<Exception> executionErrors) {
final StringBuilder finalText = new StringBuilder();
finalText.append("Regions re-added into Meta:
").append(readdedRegionNames.size());
if(!readdedRegionNames.isEmpty()){
@@ -886,6 +938,20 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
return finalText.toString();
}
+ private String formatRemovedRegionsMessage(int totalRemoved,
+ List<Exception> executionErrors) {
+ final StringBuilder finalText = new StringBuilder();
+ finalText.append("Regions that had no dir on the FileSystem and got
removed from Meta: ").
+ append(totalRemoved);
+ if(!executionErrors.isEmpty()){
+ finalText.append("\n")
+ .append("ERROR: \n\t")
+ .append("There were following errors on at least one table thread:\n");
+ executionErrors.forEach(e ->
finalText.append(e.getMessage()).append("\n"));
+ }
+ return finalText.toString();
+ }
+
private String buildHbck2AssignsCommand(List<String> regions) {
final StringBuilder builder = new StringBuilder();
builder.append("assigns ");
diff --git
a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
index ee1c08c..551c4e9 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
@@ -18,6 +18,9 @@
package org.apache.hbase;
import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -67,8 +71,11 @@ public class TestFsRegionsMetaRecoverer {
this.mockedFileSystem = Mockito.mock(FileSystem.class);
this.mockedTable = Mockito.mock(Table.class);
Configuration config = HBaseConfiguration.create();
- Mockito.when(this.mockedConnection.getConfiguration()).thenReturn(config);
-
Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)).thenReturn(mockedTable);
+ when(this.mockedConnection.getConfiguration()).thenReturn(config);
+
when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)).thenReturn(mockedTable);
+ Admin mockedAdmin = Mockito.mock(Admin.class);
+ when(this.mockedConnection.getAdmin()).thenReturn(mockedAdmin);
+
when(mockedAdmin.tableExists(TableName.valueOf("test-tbl"))).thenReturn(true);
this.testTblDir = config.get(HConstants.HBASE_DIR) +
"/data/default/test-tbl";
this.fixer = new FsRegionsMetaRecoverer(config, mockedConnection,
mockedFileSystem);
}
@@ -108,17 +115,7 @@ public class TestFsRegionsMetaRecoverer {
@Test
public void testFindMissingRegionsInMETANoMissing() throws Exception {
- ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
-
Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
- RegionInfo info = createRegionInfo("test-tbl");
- List<Cell> cells = new ArrayList<>();
- cells.add(createCellForRegionInfo(info));
- Result result = Result.create(cells);
- Mockito.when(mockedRS.next()).thenReturn(result,(Result)null);
- FileStatus status = new FileStatus();
- status.setPath(new Path(this.testTblDir + "/" + info.getEncodedName()));
- Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
- .thenReturn(new FileStatus[]{status});
+ createRegionInMetaAndFileSystem();
assertEquals("Should had returned 0 missing regions",
0, fixer.findMissingRegionsInMETA("test-tbl").size());
}
@@ -126,13 +123,13 @@ public class TestFsRegionsMetaRecoverer {
@Test
public void testFindMissingRegionsInMETAOneMissing() throws Exception {
ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
-
Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ when(this.mockedTable.getScanner(any(Scan.class))).thenReturn(mockedRS);
List<Cell> cells = new ArrayList<>();
Result result = Result.create(cells);
- Mockito.when(mockedRS.next()).thenReturn(result,(Result)null);
+ when(mockedRS.next()).thenReturn(result,(Result)null);
Path p = new Path(this.testTblDir+ "/182182182121");
FileStatus status = new FileStatus(0, true, 0, 0,0, p);
- Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
.thenReturn(new FileStatus[]{status});
List<Path> missingRegions = fixer.findMissingRegionsInMETA("test-tbl");
assertEquals("Should had returned 1 missing region",
@@ -145,33 +142,97 @@ public class TestFsRegionsMetaRecoverer {
RegionInfo info = this.createRegionInfo("test-tbl");
Path regionPath = new Path("/hbase/data/default/test-tbl/" +
info.getEncodedName());
FSDataInputStream fis = new FSDataInputStream(new
TestInputStreamSeekable(info));
- Mockito.when(this.mockedFileSystem.open(new Path(regionPath,
".regioninfo")))
+ when(this.mockedFileSystem.open(new Path(regionPath, ".regioninfo")))
.thenReturn(fis);
fixer.putRegionInfoFromHdfsInMeta(regionPath);
Mockito.verify(this.mockedConnection).getTable(TableName.META_TABLE_NAME);
- Mockito.verify(this.mockedTable).put(Mockito.any(Put.class));
+ Mockito.verify(this.mockedTable).put(any(Put.class));
}
@Test
public void testReportTablesMissingRegionsOneMissing() throws Exception {
ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
-
Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ when(this.mockedTable.getScanner(any(Scan.class))).thenReturn(mockedRS);
List<Cell> cells = new ArrayList<>();
cells.add(createCellForTableState(TableName.valueOf("test-tbl")));
Result result = Result.create(cells);
- Mockito.when(mockedRS.next()).thenReturn(result,(Result)null);
+ when(mockedRS.next()).thenReturn(result,(Result)null);
FileStatus status = new FileStatus();
Path p = new Path(this.testTblDir+ "/182182182121");
status.setPath(p);
- Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
.thenReturn(new FileStatus[]{status});
- Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME))
- .thenReturn(this.mockedTable);
Map<TableName, List<Path>> report = fixer.reportTablesMissingRegions(null);
assertEquals("Should had returned 1 missing region",
1,report.size());
}
+ @Test
+ public void testFindExtraRegionsInMETANoExtra() throws Exception {
+ createRegionInMetaAndFileSystem();
+ assertEquals("Should had returned 0 extra regions",
+ 0, fixer.findExtraRegionsInMETA("test-tbl").size());
+ }
+
+ private void createRegionInMetaAndFileSystem() throws Exception{
+ RegionInfo info = createRegionInMeta(Mockito.mock(ResultScanner.class));
+ FileStatus status = new FileStatus(0, true, 1,128,
+ System.currentTimeMillis(), new Path(this.testTblDir + "/" +
info.getEncodedName()));
+ when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ .thenReturn(new FileStatus[]{status});
+ }
+
+ private RegionInfo createRegionInMeta(ResultScanner mockedRS) throws
Exception {
+ when(this.mockedTable.getScanner(any(Scan.class))).thenReturn(mockedRS);
+ RegionInfo info = createRegionInfo("test-tbl");
+ List<Cell> cells = new ArrayList<>();
+ cells.add(createCellForRegionInfo(info));
+ Result result = Result.create(cells);
+ when(mockedRS.next()).thenReturn(result,(Result)null);
+ return info;
+ }
+
+ @Test
+ public void testFindExtraRegionsInMETAOneExtra() throws Exception {
+ RegionInfo info = createRegionInMeta(Mockito.mock(ResultScanner.class));
+ List<RegionInfo> missingRegions = fixer.findExtraRegionsInMETA("test-tbl");
+ assertEquals("Should had returned 1 extra region",
+ 1, missingRegions.size());
+ assertEquals(info.getEncodedName(),missingRegions.get(0).getEncodedName());
+ }
+
+ @Test
+ public void testRemoveRegionInfoFromMeta() throws Exception {
+ ResultScanner mockedRsTables = Mockito.mock(ResultScanner.class);
+ List<Cell> cells = new ArrayList<>();
+ cells.add(createCellForTableState(TableName.valueOf("test-tbl")));
+ Result result = Result.create(cells);
+ ResultScanner mockedRsRegions = Mockito.mock(ResultScanner.class);
+ createRegionInMeta(mockedRsRegions);
+ when(this.mockedTable.getScanner(any(Scan.class))).
+ thenReturn(mockedRsTables).
+ thenReturn(mockedRsRegions);
+ when(mockedRsTables.next()).thenReturn(result,(Result)null);
+ List<String> tableList = new ArrayList<>();
+ tableList.add("default:test-tbl");
+ fixer.removeExtraRegionsFromMetaForTables(tableList);
+ Mockito.verify(this.mockedTable).delete(anyList());
+ }
+
+
+ @Test
+ public void testReportTablesExtraRegionsOneExtra() throws Exception {
+ ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
+
Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ List<Cell> cells = new ArrayList<>();
+ cells.add(createCellForTableState(TableName.valueOf("test-tbl")));
+ Result result = Result.create(cells);
+ Mockito.when(mockedRS.next()).thenReturn(result,(Result)null);
+ Map<TableName, List<RegionInfo>> report =
fixer.reportTablesExtraRegions(null);
+ assertEquals("Should had returned 1 extra region.",
+ 1,report.size());
+ }
+
private static final class TestInputStreamSeekable extends FSInputStream {
private ByteArrayInputStream in;
private long length;
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 7b9bf2a..a095040 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -18,6 +18,7 @@
package org.apache.hbase;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -27,6 +28,8 @@ import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.Path;
@@ -45,8 +48,10 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -78,7 +83,6 @@ public class TestHBCK2 {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5);
}
@AfterClass
@@ -87,8 +91,14 @@ public class TestHBCK2 {
}
@Before
- public void before() {
+ public void before() throws Exception {
this.hbck2 = new HBCK2(TEST_UTIL.getConfiguration());
+ TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5);
+ }
+
+ @After
+ public void after() throws Exception {
+ TEST_UTIL.deleteTable(TABLE_NAME);
}
@Test (expected = UnsupportedOperationException.class)
@@ -229,10 +239,13 @@ public class TestHBCK2 {
@Test
public void testFormatReportMissingRegionsInMetaNoMissing() throws
IOException {
- final String expectedResult = "Missing Regions for each table:\n"
- + "\tTestHBCK2 -> No missing regions\n\thbase:namespace -> No missing
regions\n\t\n";
+ String expectedResult = "Missing Regions for each table:\n";
String result = testFormatMissingRegionsInMetaReport();
assertTrue(result.contains(expectedResult));
+ expectedResult = "\thbase:namespace -> No mismatching regions. This table
is good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ expectedResult = "TestHBCK2 -> No mismatching regions. This table is
good!\n\t";
+ assertTrue(result.contains(expectedResult));
}
@Test
@@ -250,7 +263,7 @@ public class TestHBCK2 {
+ regions.get(0).getEncodedName();
assertTrue(result.contains(expectedResult));
//validates namespace region is not reported missing
- expectedResult = "\n\thbase:namespace -> No missing regions\n\t";
+ expectedResult = "\n\thbase:namespace -> No mismatching regions. This
table is good!\n\t";
assertTrue(result.contains(expectedResult));
}
@@ -288,8 +301,18 @@ public class TestHBCK2 {
int remaining = totalRegions - missingRegions;
assertEquals("Table should have " + remaining + " regions in META.",
remaining,
MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
-
assertEquals(missingRegions,hbck.addMissingRegionsInMetaForTables("default:"
- + tableName.getNameAsString()).getFirst().size());
+ List<Future<List<String>>> result =
hbck.addMissingRegionsInMetaForTables("default:" +
+ tableName.getNameAsString());
+
+ Integer total = result.stream().map(f -> {
+ try {
+ return f.get().size();
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }).reduce(0, Integer::sum);
+ assertEquals(missingRegions, total.intValue());
assertEquals("Table regions should had been re-added in META.",
totalRegions,
MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
//compare the added regions to make sure those are the same
@@ -349,4 +372,194 @@ public class TestHBCK2 {
fail(e.getMessage());
}
}
+
+ private void deleteRegionDir(TableName tableName, String regionEncodedName) {
+ try {
+ Path tableDir =
FSUtils.getTableDir(this.TEST_UTIL.getDataTestDirOnTestFS(), tableName);
+ Path regionPath = new Path(tableDir, regionEncodedName);
+ this.TEST_UTIL.getTestFileSystem().delete(regionPath, true);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRemoveExtraRegionsInMetaTwoExtras() throws Exception {
+ this.testRemoveExtraRegionsInMetaForTables(2,5);
+ }
+
+ @Test
+ public void testReportExtraRegionsInMetaAllNsTbls() throws Exception {
+ String[] nullArgs = null;
+ this.testReportExtraRegionsInMeta(5, 5,
+ nullArgs);
+ }
+
+ @Test
+ public void testReportExtraRegionsInMetaSpecificTbl() throws Exception {
+ this.testReportExtraRegionsInMeta(5, 5,
+ TABLE_NAME.getNameWithNamespaceInclAsString());
+ }
+
+ @Test
+ public void testReportExtraRegionsInMetaSpecificTblAndNsTbl() throws
Exception {
+ this.testReportExtraRegionsInMeta(5, 5,
+ TABLE_NAME.getNameWithNamespaceInclAsString(), "hbase:namespace");
+ }
+
+ @Test
+ public void testReportExtraRegionsInMetaSpecificTblAndNsTblAlsoExtra()
throws Exception {
+ TableName tableName = createTestTable(5);
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ deleteRegionDir(tableName, regions.get(0).getEncodedName());
+ this.testReportExtraRegionsInMeta(5, 6,
+ TABLE_NAME.getNameWithNamespaceInclAsString(),
+ tableName.getNameWithNamespaceInclAsString());
+ }
+
+ @Test
+ public void testFormatReportExtraRegionsInMetaNoExtra() throws IOException {
+ String expectedResult = "Regions in Meta but having no equivalent dir, for
each table:\n";
+ String result = testFormatExtraRegionsInMetaReport();
+ assertTrue(result.contains(expectedResult));
+ expectedResult = "\thbase:namespace -> No mismatching regions. This table
is good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ expectedResult = "TestHBCK2 -> No mismatching regions. This table is
good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ }
+
+ @Test
+ public void testFormatReportExtraInMetaOneExtra() throws IOException {
+ TableName tableName = createTestTable(5);
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ deleteRegionDir(tableName, regions.get(0).getEncodedName());
+ String expectedResult = "Regions in Meta but having no equivalent dir, for
each table:\n";
+ String result = testFormatExtraRegionsInMetaReport();
+ //validates initial report message
+ assertTrue(result.contains(expectedResult));
+ //validates our test table region is reported as extra
+ expectedResult = "\t" + tableName.getNameAsString() + "->\n\t\t"
+ + regions.get(0).getEncodedName();
+ assertTrue(result.contains(expectedResult));
+ //validates namespace region is not reported missing
+ expectedResult = "\n\thbase:namespace -> No mismatching regions. This
table is good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ }
+
+ @Test
+ public void testFormatFixExtraRegionsInMetaNoExtra() throws IOException {
+ String expectedResult = "Regions in Meta but having no equivalent dir, for
each table:\n";
+ String result = testFormatExtraRegionsInMetaFix(null);
+ assertTrue(result.contains(expectedResult));
+ expectedResult = "\thbase:namespace -> No mismatching regions. This table
is good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ expectedResult = "TestHBCK2 -> No mismatching regions. This table is
good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ }
+
+ @Test
+ public void testFormatFixExtraRegionsInMetaNoExtraSpecifyTable() throws
IOException {
+ final String expectedResult = "Regions in Meta but having no equivalent
dir, for each table:\n"
+ + "\thbase:namespace -> No mismatching regions. This table is good!\n\t";
+ String result = testFormatExtraRegionsInMetaFix("hbase:namespace");
+ assertTrue(result.contains(expectedResult));
+ }
+
+ @Test
+ public void testFormatFixExtraInMetaOneExtra() throws IOException {
+ TableName tableName = createTestTable(5);
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ deleteRegionDir(tableName, regions.get(0).getEncodedName());
+ String expectedResult = "Regions in Meta but having no equivalent dir, for
each table:\n";
+ String result = testFormatExtraRegionsInMetaFix(null);
+ //validates initial execute message
+ assertTrue(result.contains(expectedResult));
+ //validates our test table region is reported as extra
+ expectedResult = "\t" + tableName.getNameAsString() + "->\n\t\t"
+ + regions.get(0).getEncodedName();
+ assertTrue(result.contains(expectedResult));
+ //validates namespace region is not reported missing
+ expectedResult = "\n\thbase:namespace -> No mismatching regions. This
table is good!\n\t";
+ assertTrue(result.contains(expectedResult));
+ }
+
+ @Test
+ public void testFormatFixExtraInMetaOneExtraSpecificTable() throws
IOException {
+ TableName tableName = createTestTable(5);
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ deleteRegionDir(tableName, regions.get(0).getEncodedName());
+ String expectedResult = "Regions in Meta but having no equivalent dir, for
each table:\n";
+ String result =
testFormatExtraRegionsInMetaFix(tableName.getNameWithNamespaceInclAsString());
+ //validates initial execute message
+ assertTrue(result.contains(expectedResult));
+ //validates our test table region is reported as extra
+ expectedResult = "\t" + tableName.getNameAsString() + "->\n\t\t"
+ + regions.get(0).getEncodedName();
+ assertTrue(result.contains(expectedResult));
+ //validates namespace region is not reported missing
+ expectedResult = "\n\thbase:namespace -> No mismatching regions. This
table is good!\n\t";
+ assertFalse("Should not contain: " + expectedResult,
result.contains(expectedResult));
+ }
+
+ private String testFormatExtraRegionsInMetaReport() throws IOException {
+ return testFormatExtraRegionsInMeta(new
String[]{HBCK2.EXTRA_REGIONS_IN_META });
+ }
+
+ private String testFormatExtraRegionsInMetaFix(String table) throws
IOException {
+ if(table!=null) {
+ return testFormatExtraRegionsInMeta(new String[]
{HBCK2.EXTRA_REGIONS_IN_META, "-f", table});
+ } else {
+ return testFormatExtraRegionsInMeta(new String[]
{HBCK2.EXTRA_REGIONS_IN_META, "-f"});
+ }
+ }
+
+ private String testFormatExtraRegionsInMeta(String[] args) throws
IOException {
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ final StringBuilder builder = new StringBuilder();
+ PrintStream originalOS = System.out;
+ OutputStream testOS = new OutputStream() {
+ @Override public void write(int b) throws IOException {
+ builder.append((char)b);
+ }
+ };
+ System.setOut(new PrintStream(testOS));
+ hbck.run(args);
+ System.setOut(originalOS);
+ return builder.toString();
+ }
+
+ private void testRemoveExtraRegionsInMetaForTables(int extraRegions, int
totalRegions)
+ throws Exception {
+ TableName tableName = createTestTable(totalRegions);
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ regions.subList(0, extraRegions).forEach(r -> deleteRegionDir(tableName,
r.getEncodedName()));
+ int remaining = totalRegions - extraRegions;
+ assertEquals(extraRegions, hbck.extraRegionsInMeta(new String[]
+ { "-f",
+ "default:" + tableName.getNameAsString()
+ }).get(tableName).size());
+ assertEquals("Table regions should had been removed from META.", remaining,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
+ }
+
+ private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl,
+ int expectedTotalExtraRegions, String... namespaceOrTable) throws
Exception {
+ List<RegionInfo> regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME);
+ regions.subList(0, extraRegionsInTestTbl).forEach(r ->
deleteRegionDir(TABLE_NAME,
+ r.getEncodedName()));
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ final Map<TableName,List<String>> report =
+ hbck.extraRegionsInMeta(namespaceOrTable);
+ long resultingExtraRegions = report.keySet().stream().mapToLong(nsTbl ->
+ report.get(nsTbl).size()).sum();
+ assertEquals(expectedTotalExtraRegions, resultingExtraRegions);
+ }
+
}
