Repository: hbase Updated Branches: refs/heads/branch-1.0 9e18d8aa5 -> c86607fd7
HBASE-12223 Revert from 1.0 branch Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c86607fd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c86607fd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c86607fd Branch: refs/heads/branch-1.0 Commit: c86607fd71a86904592a0c1789872fd40e2dafdb Parents: 9e18d8a Author: tedyu <[email protected]> Authored: Thu Dec 18 10:55:02 2014 -0800 Committer: tedyu <[email protected]> Committed: Thu Dec 18 10:55:02 2014 -0800 ---------------------------------------------------------------------- .../mapreduce/MultiTableInputFormatBase.java | 101 ++++++++----------- 1 file changed, 41 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c86607fd/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java index b9a2db7..5c253cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java @@ -46,9 +46,6 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import java.util.Map; -import java.util.HashMap; -import java.util.Iterator; /** * A base for {@link MultiTableInputFormat}s. Receives a list of * {@link Scan} instances that define the input tables and @@ -132,82 +129,67 @@ public abstract class MultiTableInputFormatBase extends if (scans.isEmpty()) { throw new IOException("No scans were provided."); } + List<InputSplit> splits = new ArrayList<InputSplit>(); - Map<TableName, List<Scan>> tableMaps = new HashMap<TableName, List<Scan>>(); for (Scan scan : scans) { byte[] tableNameBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME); if (tableNameBytes == null) throw new IOException("A scan object did not have a table name"); TableName tableName = TableName.valueOf(tableNameBytes); - - List<Scan> scanList = tableMaps.get(tableName); - if (scanList == null) { - scanList = new ArrayList<Scan>(); - tableMaps.put(tableName, scanList); - } - scanList.add(scan); - } - - List<InputSplit> splits = new ArrayList<InputSplit>(); - Iterator iter = tableMaps.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry<TableName, List<Scan>> entry = (Map.Entry<TableName, List<Scan>>) iter.next(); - TableName tableName = entry.getKey(); - List<Scan> scanList = entry.getValue(); Table table = null; RegionLocator regionLocator = null; Connection conn = null; - - try{ + try { conn = ConnectionFactory.createConnection(context.getConfiguration()); table = conn.getTable(tableName); + regionLocator = conn.getRegionLocator(tableName); regionLocator = (RegionLocator) table; - RegionSizeCalculator sizeCalculator = new RegionSizeCalculator( - regionLocator, conn.getAdmin()); Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys(); - for (Scan scan : scanList) { - if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { - throw new IOException("Expecting at least one region for table : " - + tableName.getNameAsString()); - } - int count = 0; - - byte[] startRow = scan.getStartRow(); - byte[] stopRow = scan.getStopRow(); + if (keys == null || keys.getFirst() == null || + keys.getFirst().length == 0) { + throw new IOException("Expecting at least one region for table : " + + tableName.getNameAsString()); + } + int count = 0; - for (int i = 0; i < keys.getFirst().length; i++) { - if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) { - continue; - } + byte[] startRow = scan.getStartRow(); + byte[] stopRow = scan.getStopRow(); - if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || - Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && - (stopRow.length == 0 || Bytes.compareTo(stopRow, - keys.getFirst()[i]) > 0)) { - byte[] splitStart = startRow.length == 0 || - Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? - keys.getFirst()[i] : startRow; - byte[] splitStop = (stopRow.length == 0 || - Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && - keys.getSecond()[i].length > 0 ? - keys.getSecond()[i] : stopRow; + RegionSizeCalculator sizeCalculator = new RegionSizeCalculator( + regionLocator, conn.getAdmin()); - HRegionLocation hregionLocation = regionLocator.getRegionLocation( - keys.getFirst()[i], false); - String regionHostname = hregionLocation.getHostname(); - HRegionInfo regionInfo = hregionLocation.getRegionInfo(); - long regionSize = sizeCalculator.getRegionSize( - regionInfo.getRegionName()); + for (int i = 0; i < keys.getFirst().length; i++) { + if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) { + continue; + } + HRegionLocation hregionLocation = regionLocator.getRegionLocation( + keys.getFirst()[i], false); + String regionHostname = hregionLocation.getHostname(); + HRegionInfo regionInfo = hregionLocation.getRegionInfo(); - TableSplit split = new TableSplit(table.getName(), - scan, splitStart, splitStop, regionHostname, regionSize); + // determine if the given start and stop keys fall into the range + if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || + Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && + (stopRow.length == 0 || + Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { + byte[] splitStart = + startRow.length == 0 || + Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys + .getFirst()[i] : startRow; + byte[] splitStop = + (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], + stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys + .getSecond()[i] : stopRow; - splits.add(split); + long regionSize = sizeCalculator.getRegionSize(regionInfo.getRegionName()); + TableSplit split = + new TableSplit(regionLocator.getName(), + scan, splitStart, splitStop, regionHostname, regionSize); - if (LOG.isDebugEnabled()) - LOG.debug("getSplits: split -> " + (count++) + " -> " + split); - } + splits.add(split); + if (LOG.isDebugEnabled()) + LOG.debug("getSplits: split -> " + (count++) + " -> " + split); } } } finally { @@ -216,7 +198,6 @@ public abstract class MultiTableInputFormatBase extends if (null != conn) conn.close(); } } - return splits; }
