Repository: hbase Updated Branches: refs/heads/branch-2.1 147694bb0 -> 4340930c7
HBASE-20649 Validate HFiles do not have PREFIX_TREE DataBlockEncoding; ADDEDNDUM ADD MISSING FILE Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4340930c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4340930c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4340930c Branch: refs/heads/branch-2.1 Commit: 4340930c716599256c823c78d614b4062ec3457c Parents: 147694b Author: Michael Stack <st...@apache.org> Authored: Tue Aug 28 07:45:27 2018 -0700 Committer: Michael Stack <st...@apache.org> Committed: Tue Aug 28 07:45:27 2018 -0700 ---------------------------------------------------------------------- .../hbase/tool/HFileContentValidator.java | 126 +++++++++++++++++++ 1 file changed, 126 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4340930c/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java new file mode 100644 index 0000000..d60844b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java @@ -0,0 +1,126 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.tool; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; + +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +public class HFileContentValidator extends AbstractHBaseTool { + + private static final Logger LOG = LoggerFactory.getLogger(HFileContentValidator.class); + + /** + * Check HFile contents are readable by HBase 2. + * + * @param conf used configuration + * @return number of HFiles corrupted HBase + * @throws IOException if a remote or network exception occurs + */ + private boolean validateHFileContent(Configuration conf) throws IOException { + FileSystem fileSystem = FSUtils.getCurrentFileSystem(conf); + + ExecutorService threadPool = createThreadPool(conf); + HFileCorruptionChecker checker; + + try { + checker = new HFileCorruptionChecker(conf, threadPool, false); + + Path rootDir = FSUtils.getRootDir(conf); + LOG.info("Validating HFile contents under {}", rootDir); + + Collection<Path> tableDirs = FSUtils.getTableDirs(fileSystem, rootDir); + checker.checkTables(tableDirs); + + Path archiveRootDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + LOG.info("Validating HFile contents under {}", archiveRootDir); + + List<Path> archiveTableDirs = FSUtils.getTableDirs(fileSystem, archiveRootDir); + checker.checkTables(archiveTableDirs); + } finally { + threadPool.shutdown(); + + try { + threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + int checkedFiles = checker.getHFilesChecked(); + Collection<Path> corrupted = checker.getCorrupted(); + + if (corrupted.isEmpty()) { + LOG.info("Checked {} HFiles, none of them are corrupted.", checkedFiles); + LOG.info("There are no incompatible HFiles."); + + return true; + } else { + LOG.info("Checked {} HFiles, {} are corrupted.", checkedFiles, corrupted.size()); + + for (Path path : corrupted) { + LOG.info("Corrupted file: {}", path); + } + + LOG.info("Change data block encodings before upgrading. " + + "Check https://s.apache.org/prefixtree for instructions."); + + return false; + } + } + + private ExecutorService createThreadPool(Configuration conf) { + int availableProcessors = Runtime.getRuntime().availableProcessors(); + int numThreads = conf.getInt("hfilevalidator.numthreads", availableProcessors); + return Executors.newFixedThreadPool(numThreads, + Threads.getNamedThreadFactory("hfile-validator")); + } + + @Override + protected void addOptions() { + } + + @Override + protected void processOptions(CommandLine cmd) { + } + + @Override + protected int doWork() throws Exception { + return (validateHFileContent(getConf())) ? EXIT_SUCCESS : EXIT_FAILURE; + } +}