HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/241ad0d7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/241ad0d7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/241ad0d7 Branch: refs/heads/branch-1.0 Commit: 241ad0d767d0d53d8777ca2f4e77ff04c4b04216 Parents: f87aab7 Author: Matteo Bertozzi <[email protected]> Authored: Thu Jan 15 23:13:47 2015 +0000 Committer: Matteo Bertozzi <[email protected]> Committed: Thu Jan 15 23:15:40 2015 +0000 ---------------------------------------------------------------------- .../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++-- .../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/241ad0d7/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index b4b6adc..55d20f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -204,8 +204,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool { } Path familyDir = stat.getPath(); byte[] family = familyDir.getName().getBytes(); - Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); - for (Path hfile : hfiles) { + FileStatus[] hfileStatuses = fs.listStatus(familyDir); + for (FileStatus hfileStatus : hfileStatuses) { + long length = hfileStatus.getLen(); + Path hfile = hfileStatus.getPath(); // Skip "_", reference, HFileLink String fileName = hfile.getName(); if (fileName.startsWith("_")) continue; @@ -217,6 +219,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool { LOG.warn("Skipping HFileLink " + fileName); continue; } + if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " + + length + " bytes can be problematic as it may lead to oversplitting."); + } ret.add(new LoadQueueItem(family, hfile)); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/241ad0d7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index ad701b7..6a65038 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -709,6 +709,12 @@ public class HStore implements Store { + this.getRegionInfo().getRegionNameAsString()); } + if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " + + reader.length() + " bytes can be problematic as it may lead to oversplitting."); + } + if (verifyBulkLoads) { long verificationStartTime = EnvironmentEdgeManager.currentTime(); LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
