Repository: hbase Updated Branches: refs/heads/0.98 5e36e70cf -> 358e60027 refs/heads/branch-1 2e08bb3b4 -> 52b4db860 refs/heads/branch-1.0 f87aab7ec -> 241ad0d76 refs/heads/master eafc07a06 -> 621b33f2f
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/621b33f2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/621b33f2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/621b33f2 Branch: refs/heads/master Commit: 621b33f2f9677bd30c78de0ada757b4349eb017b Parents: eafc07a Author: Matteo Bertozzi <[email protected]> Authored: Thu Jan 15 23:13:47 2015 +0000 Committer: Matteo Bertozzi <[email protected]> Committed: Thu Jan 15 23:13:47 2015 +0000 ---------------------------------------------------------------------- .../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++-- .../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/621b33f2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index 2115978..45f57b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool { } Path familyDir = stat.getPath(); byte[] family = familyDir.getName().getBytes(); - Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); - for (Path hfile : hfiles) { + FileStatus[] hfileStatuses = fs.listStatus(familyDir); + for (FileStatus hfileStatus : hfileStatuses) { + long length = hfileStatus.getLen(); + Path hfile = hfileStatus.getPath(); // Skip "_", reference, HFileLink String fileName = hfile.getName(); if (fileName.startsWith("_")) continue; @@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool { LOG.warn("Skipping HFileLink " + fileName); continue; } + if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " + + length + " bytes can be problematic as it may lead to oversplitting."); + } ret.add(new LoadQueueItem(family, hfile)); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/621b33f2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index b674fea..047d689 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -709,6 +709,12 @@ public class HStore implements Store { + this.getRegionInfo().getRegionNameAsString()); } + if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " + + reader.length() + " bytes can be problematic as it may lead to oversplitting."); + } + if (verifyBulkLoads) { long verificationStartTime = EnvironmentEdgeManager.currentTime(); LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
