[01/30] hbase git commit: HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto balancing feature [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19064 e7b37cf93 -> 1e7c341cb (forced update)
HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto
balancing feature
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c7593d14
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c7593d14
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c7593d14
Branch: refs/heads/HBASE-19064
Commit: c7593d14a937509ee5d6a4dcb8322651edd9c6fd
Parents: f663430
Author: Yuki Tawara
Authored: Sun Apr 8 01:46:52 2018 +0900
Committer: tedyu
Committed: Sun Apr 8 10:58:23 2018 -0700
--
.../hbase/mapreduce/TableInputFormatBase.java | 12 +-
.../mapreduce/TestTableInputFormatBase.java | 232 +++
2 files changed, 242 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7593d14/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index d8031d9..24973c9 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.StringUtils;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* A base for {@link TableInputFormat}s. Receives a {@link Connection}, a
{@link TableName},
@@ -291,7 +292,7 @@ public abstract class TableInputFormatBase
*/
private List oneInputSplitPerRegion() throws IOException {
RegionSizeCalculator sizeCalculator =
-new RegionSizeCalculator(getRegionLocator(), getAdmin());
+createRegionSizeCalculator(getRegionLocator(), getAdmin());
TableName tableName = getTable().getName();
@@ -478,7 +479,8 @@ public abstract class TableInputFormatBase
while (j < splits.size()) {
TableSplit nextRegion = (TableSplit) splits.get(j);
long nextRegionSize = nextRegion.getLength();
- if (totalSize + nextRegionSize <= averageRegionSize) {
+ if (totalSize + nextRegionSize <= averageRegionSize
+ && Bytes.equals(splitEndKey, nextRegion.getStartRow())) {
totalSize = totalSize + nextRegionSize;
splitEndKey = nextRegion.getEndRow();
j++;
@@ -586,6 +588,12 @@ public abstract class TableInputFormatBase
this.connection = connection;
}
+ @VisibleForTesting
+ protected RegionSizeCalculator createRegionSizeCalculator(RegionLocator
locator, Admin admin)
+ throws IOException {
+return new RegionSizeCalculator(locator, admin);
+ }
+
/**
* Gets the scan defining the actual details like columns etc.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7593d14/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
index 5fa4b54..29a92ee 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
@@ -18,15 +18,45 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Buffered
[01/22] hbase git commit: HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto balancing feature [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-20046-branch-2 e821b9ac5 -> 74ab10c35 (forced update)
HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto
balancing feature
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1ec708bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1ec708bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1ec708bf
Branch: refs/heads/HBASE-20046-branch-2
Commit: 1ec708bf6cbe6cead0270a59f0b7d1e18dd3ac38
Parents: 10ed3f4
Author: Yuki Tawara
Authored: Sun Apr 8 01:46:52 2018 +0900
Committer: tedyu
Committed: Sun Apr 8 10:58:56 2018 -0700
--
.../hbase/mapreduce/TableInputFormatBase.java | 12 +-
.../mapreduce/TestTableInputFormatBase.java | 232 +++
2 files changed, 242 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1ec708bf/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index d8031d9..24973c9 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.StringUtils;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* A base for {@link TableInputFormat}s. Receives a {@link Connection}, a
{@link TableName},
@@ -291,7 +292,7 @@ public abstract class TableInputFormatBase
*/
private List oneInputSplitPerRegion() throws IOException {
RegionSizeCalculator sizeCalculator =
-new RegionSizeCalculator(getRegionLocator(), getAdmin());
+createRegionSizeCalculator(getRegionLocator(), getAdmin());
TableName tableName = getTable().getName();
@@ -478,7 +479,8 @@ public abstract class TableInputFormatBase
while (j < splits.size()) {
TableSplit nextRegion = (TableSplit) splits.get(j);
long nextRegionSize = nextRegion.getLength();
- if (totalSize + nextRegionSize <= averageRegionSize) {
+ if (totalSize + nextRegionSize <= averageRegionSize
+ && Bytes.equals(splitEndKey, nextRegion.getStartRow())) {
totalSize = totalSize + nextRegionSize;
splitEndKey = nextRegion.getEndRow();
j++;
@@ -586,6 +588,12 @@ public abstract class TableInputFormatBase
this.connection = connection;
}
+ @VisibleForTesting
+ protected RegionSizeCalculator createRegionSizeCalculator(RegionLocator
locator, Admin admin)
+ throws IOException {
+return new RegionSizeCalculator(locator, admin);
+ }
+
/**
* Gets the scan defining the actual details like columns etc.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/1ec708bf/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
index 5fa4b54..29a92ee 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
@@ -18,15 +18,45 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hba
hbase git commit: HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto balancing feature
Repository: hbase
Updated Branches:
refs/heads/branch-2 10ed3f41f -> 1ec708bf6
HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto
balancing feature
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1ec708bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1ec708bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1ec708bf
Branch: refs/heads/branch-2
Commit: 1ec708bf6cbe6cead0270a59f0b7d1e18dd3ac38
Parents: 10ed3f4
Author: Yuki Tawara
Authored: Sun Apr 8 01:46:52 2018 +0900
Committer: tedyu
Committed: Sun Apr 8 10:58:56 2018 -0700
--
.../hbase/mapreduce/TableInputFormatBase.java | 12 +-
.../mapreduce/TestTableInputFormatBase.java | 232 +++
2 files changed, 242 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1ec708bf/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index d8031d9..24973c9 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.StringUtils;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* A base for {@link TableInputFormat}s. Receives a {@link Connection}, a
{@link TableName},
@@ -291,7 +292,7 @@ public abstract class TableInputFormatBase
*/
private List oneInputSplitPerRegion() throws IOException {
RegionSizeCalculator sizeCalculator =
-new RegionSizeCalculator(getRegionLocator(), getAdmin());
+createRegionSizeCalculator(getRegionLocator(), getAdmin());
TableName tableName = getTable().getName();
@@ -478,7 +479,8 @@ public abstract class TableInputFormatBase
while (j < splits.size()) {
TableSplit nextRegion = (TableSplit) splits.get(j);
long nextRegionSize = nextRegion.getLength();
- if (totalSize + nextRegionSize <= averageRegionSize) {
+ if (totalSize + nextRegionSize <= averageRegionSize
+ && Bytes.equals(splitEndKey, nextRegion.getStartRow())) {
totalSize = totalSize + nextRegionSize;
splitEndKey = nextRegion.getEndRow();
j++;
@@ -586,6 +588,12 @@ public abstract class TableInputFormatBase
this.connection = connection;
}
+ @VisibleForTesting
+ protected RegionSizeCalculator createRegionSizeCalculator(RegionLocator
locator, Admin admin)
+ throws IOException {
+return new RegionSizeCalculator(locator, admin);
+ }
+
/**
* Gets the scan defining the actual details like columns etc.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/1ec708bf/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
index 5fa4b54..29a92ee 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
@@ -18,15 +18,45 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.BufferedMutatorParams;
+import
hbase git commit: HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto balancing feature
Repository: hbase
Updated Branches:
refs/heads/master f66343050 -> c7593d14a
HBASE-20361 Non-successive TableInputSplits may wrongly be merged by auto
balancing feature
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c7593d14
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c7593d14
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c7593d14
Branch: refs/heads/master
Commit: c7593d14a937509ee5d6a4dcb8322651edd9c6fd
Parents: f663430
Author: Yuki Tawara
Authored: Sun Apr 8 01:46:52 2018 +0900
Committer: tedyu
Committed: Sun Apr 8 10:58:23 2018 -0700
--
.../hbase/mapreduce/TableInputFormatBase.java | 12 +-
.../mapreduce/TestTableInputFormatBase.java | 232 +++
2 files changed, 242 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7593d14/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index d8031d9..24973c9 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.StringUtils;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* A base for {@link TableInputFormat}s. Receives a {@link Connection}, a
{@link TableName},
@@ -291,7 +292,7 @@ public abstract class TableInputFormatBase
*/
private List oneInputSplitPerRegion() throws IOException {
RegionSizeCalculator sizeCalculator =
-new RegionSizeCalculator(getRegionLocator(), getAdmin());
+createRegionSizeCalculator(getRegionLocator(), getAdmin());
TableName tableName = getTable().getName();
@@ -478,7 +479,8 @@ public abstract class TableInputFormatBase
while (j < splits.size()) {
TableSplit nextRegion = (TableSplit) splits.get(j);
long nextRegionSize = nextRegion.getLength();
- if (totalSize + nextRegionSize <= averageRegionSize) {
+ if (totalSize + nextRegionSize <= averageRegionSize
+ && Bytes.equals(splitEndKey, nextRegion.getStartRow())) {
totalSize = totalSize + nextRegionSize;
splitEndKey = nextRegion.getEndRow();
j++;
@@ -586,6 +588,12 @@ public abstract class TableInputFormatBase
this.connection = connection;
}
+ @VisibleForTesting
+ protected RegionSizeCalculator createRegionSizeCalculator(RegionLocator
locator, Admin admin)
+ throws IOException {
+return new RegionSizeCalculator(locator, admin);
+ }
+
/**
* Gets the scan defining the actual details like columns etc.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7593d14/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
index 5fa4b54..29a92ee 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java
@@ -18,15 +18,45 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.BufferedMutatorParams;
+import org
