[04/16] hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b854859
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b854859
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b854859
Branch: refs/heads/HBASE-21512
Commit: 3b854859f6fad44cbf31164374569a6ab23f3623
Parents: f49baf2
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 10:55:42 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java| 50
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4aff949..9c94990 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -126,7 +126,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 3fbddf2..d9008b2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -186,6 +186,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..037b13e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -42,6 +48,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
@@ -98,6 +105,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set getStreamReaders() {
+List stores = REGION.getStores();
+Assert.assertEquals(1, stores.size());
+HStore firstStore = stores.get(0);
+Assert.assertNotNull(firstStore);
+Collection storeFiles =
[05/16] hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67ab8b88
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67ab8b88
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67ab8b88
Branch: refs/heads/HBASE-21512
Commit: 67ab8b888f8b393979624a2bd7d527fefd9dd6d7
Parents: 3b85485
Author: huzheng
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:26:52 2018 +0800
--
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/67ab8b88/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 037b13e..c1cecf8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -47,9 +47,9 @@ import
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -74,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES,
2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -99,8 +99,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
[43/51] [abbrv] hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b854859
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b854859
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b854859
Branch: refs/heads/HBASE-20952
Commit: 3b854859f6fad44cbf31164374569a6ab23f3623
Parents: f49baf2
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 10:55:42 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java| 50
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4aff949..9c94990 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -126,7 +126,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 3fbddf2..d9008b2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -186,6 +186,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..037b13e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -42,6 +48,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
@@ -98,6 +105,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set getStreamReaders() {
+List stores = REGION.getStores();
+Assert.assertEquals(1, stores.size());
+HStore firstStore = stores.get(0);
+Assert.assertNotNull(firstStore);
+Collection storeFiles =
[44/51] [abbrv] hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67ab8b88
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67ab8b88
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67ab8b88
Branch: refs/heads/HBASE-20952
Commit: 67ab8b888f8b393979624a2bd7d527fefd9dd6d7
Parents: 3b85485
Author: huzheng
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:26:52 2018 +0800
--
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/67ab8b88/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 037b13e..c1cecf8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -47,9 +47,9 @@ import
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -74,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES,
2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -99,8 +99,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 19ad63c16 -> 851606809
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/85160680
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/85160680
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/85160680
Branch: refs/heads/branch-2.0
Commit: 85160680966b2f221b782777fd87611625f23a07
Parents: 19ad63c
Author: huzheng
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:43:09 2018 +0800
--
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/85160680/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 6e12fbe..7bdf2aa 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -44,9 +44,9 @@ import
org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -70,8 +70,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES,
2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -95,8 +95,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 e9b0d7379 -> 0b181af4e
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b181af4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b181af4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b181af4
Branch: refs/heads/branch-2.1
Commit: 0b181af4ec45fc5eb5b2448a24e461d2c98b6448
Parents: e9b0d73
Author: huzheng
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:28:28 2018 +0800
--
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b181af4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 037b13e..c1cecf8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -47,9 +47,9 @@ import
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -74,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES,
2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -99,8 +99,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Repository: hbase
Updated Branches:
refs/heads/master 3b854859f -> 67ab8b888
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67ab8b88
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67ab8b88
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67ab8b88
Branch: refs/heads/master
Commit: 67ab8b888f8b393979624a2bd7d527fefd9dd6d7
Parents: 3b85485
Author: huzheng
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:26:52 2018 +0800
--
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/67ab8b88/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 037b13e..c1cecf8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -47,9 +47,9 @@ import
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -74,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES,
2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -99,8 +99,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
Repository: hbase
Updated Branches:
refs/heads/branch-2 96f8e0cbe -> 1a1a65b56
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1a1a65b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1a1a65b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1a1a65b5
Branch: refs/heads/branch-2
Commit: 1a1a65b565354e4a6349db3be22dd5630cf60179
Parents: 96f8e0c
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:20:38 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 +
.../regionserver/TestSwitchToStreamRead.java| 62 ++--
3 files changed, 61 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a1a65b5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4aff949..9c94990 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -126,7 +126,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a1a65b5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 3fbddf2..d9008b2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -186,6 +186,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a1a65b5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..c1cecf8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -41,8 +47,9 @@ import
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -67,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 361dea85c -> 19ad63c16
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/19ad63c1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/19ad63c1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/19ad63c1
Branch: refs/heads/branch-2.0
Commit: 19ad63c16b73d9cbd1fa8232e493319f7e12ebfe
Parents: 361dea8
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 11:02:49 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java| 50
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/19ad63c1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4a0c66f..17789d4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -123,7 +123,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/19ad63c1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index b500abf..0a4efb0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -181,6 +181,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/19ad63c1/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 0af2970..6e12fbe 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,12 +38,14 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -94,6 +101,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set getStreamReaders() {
+List stores = REGION.getStores();
+Assert.assertEquals(1, stores.size());
+HStore firstStore = stores.get(0);
+
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 58cfed60e -> e9b0d7379
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e9b0d737
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e9b0d737
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e9b0d737
Branch: refs/heads/branch-2.1
Commit: e9b0d737937e2b8557c5bc17feea3b820e282c19
Parents: 58cfed6
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 10:58:04 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java| 50
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e9b0d737/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4a0c66f..17789d4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -123,7 +123,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e9b0d737/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index b500abf..0a4efb0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -181,6 +181,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e9b0d737/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..037b13e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -42,6 +48,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
@@ -98,6 +105,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set getStreamReaders() {
+List stores = REGION.getStores();
+Assert.assertEquals(1, stores.size());
+HStore firstStore =
hbase git commit: HBASE-21551 Memory leak when use scan with STREAM at server side
Repository: hbase
Updated Branches:
refs/heads/master f49baf259 -> 3b854859f
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b854859
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b854859
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b854859
Branch: refs/heads/master
Commit: 3b854859f6fad44cbf31164374569a6ab23f3623
Parents: f49baf2
Author: huzheng
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng
Committed: Thu Dec 6 10:55:42 2018 +0800
--
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java| 50
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4aff949..9c94990 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -126,7 +126,8 @@ public class HStoreFile implements StoreFile,
StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set streamReaders;
+ @VisibleForTesting
+ final Set streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 3fbddf2..d9008b2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -186,6 +186,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+}
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..037b13e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -42,6 +48,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
@@ -98,6 +105,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set getStreamReaders() {
+List stores = REGION.getStores();
+Assert.assertEquals(1, stores.size());
+HStore firstStore =
