hbase git commit: HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can happen if masterprocwals have been cleared).
Repository: hbase
Updated Branches:
refs/heads/master cdfe80889 -> 6d7bc0e98
HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can
happen if masterprocwals have been cleared).
Add a check for hbase:meta being online before we go to read it.
If not online, move into a holding-pattern until rectified, probably
by external operator.
Incorporates bulk of patch made by Allan Yang over on HBASE-21035.
M
hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
Add a Constructor for case where retries are for ever.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Move stuff around so that the first hbase:meta read is the AM#loadMeta.
Previously, checking table state and/or favored nodes could end up
trying to read a meta that was not onlined holding up master startup.
Do similar for the namespace table. Adds new methods isMeta and
isNamespace which check that the regions/tables are online.. if not,
we wait logging with a back-off that assigns need to be run.
Signed-off-by: Allan Yang
Signed-off-by: Duo Zhang
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d7bc0e9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d7bc0e9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d7bc0e9
Branch: refs/heads/master
Commit: 6d7bc0e98b25215e79f67f107fd0d3306dfcf352
Parents: cdfe808
Author: Michael Stack
Authored: Wed Sep 12 10:47:33 2018 -0700
Committer: Michael Stack
Committed: Sun Sep 16 21:44:17 2018 -0700
--
.../hadoop/hbase/util/RetryCounterFactory.java | 4 +
.../org/apache/hadoop/hbase/master/HMaster.java | 104 +--
.../hadoop/hbase/master/MasterRpcServices.java | 5 +-
.../hadoop/hbase/master/MasterWalManager.java | 42 ++--
.../hbase/master/RegionServerTracker.java | 12 ++-
.../hbase/master/TableNamespaceManager.java | 8 +-
.../master/assignment/AssignmentManager.java| 5 +-
.../master/procedure/DisableTableProcedure.java | 2 +-
.../master/procedure/EnableTableProcedure.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 12 +--
.../hadoop/hbase/TestMetaTableAccessor.java | 16 +++
11 files changed, 177 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d7bc0e9/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
index dcf6626..c15cfb2 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
@@ -28,6 +28,10 @@ import org.apache.yetus.audience.InterfaceAudience;
public class RetryCounterFactory {
private final RetryConfig retryConfig;
+ public RetryCounterFactory(int sleepIntervalMillis) {
+this(Integer.MAX_VALUE, sleepIntervalMillis);
+ }
+
public RetryCounterFactory(int maxAttempts, int sleepIntervalMillis) {
this(maxAttempts, sleepIntervalMillis, -1);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d7bc0e9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f51496a..9f95a3a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -55,6 +55,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -199,6 +200,8 @@ import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.hadoop.hbase.util.RetryCounterFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
@@ -949,11 +952,13 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Start RegionServerTracker with listing of servers found with exiting
SCPs -- these should
// be
hbase git commit: HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can happen if masterprocwals have been cleared).
Repository: hbase
Updated Branches:
refs/heads/branch-2 7a1b4d449 -> 6616fd2e0
HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can
happen if masterprocwals have been cleared).
Add a check for hbase:meta being online before we go to read it.
If not online, move into a holding-pattern until rectified, probably
by external operator.
Incorporates bulk of patch made by Allan Yang over on HBASE-21035.
M
hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
Add a Constructor for case where retries are for ever.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Move stuff around so that the first hbase:meta read is the AM#loadMeta.
Previously, checking table state and/or favored nodes could end up
trying to read a meta that was not onlined holding up master startup.
Do similar for the namespace table. Adds new methods isMeta and
isNamespace which check that the regions/tables are online.. if not,
we wait logging with a back-off that assigns need to be run.
Signed-off-by: Allan Yang
Signed-off-by: Duo Zhang
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6616fd2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6616fd2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6616fd2e
Branch: refs/heads/branch-2
Commit: 6616fd2e0f5f88f43e02fdba25162ecbdbf11479
Parents: 7a1b4d4
Author: Michael Stack
Authored: Wed Sep 12 10:47:33 2018 -0700
Committer: Michael Stack
Committed: Sun Sep 16 21:43:48 2018 -0700
--
.../hadoop/hbase/util/RetryCounterFactory.java | 4 +
.../org/apache/hadoop/hbase/master/HMaster.java | 104 +--
.../hadoop/hbase/master/MasterRpcServices.java | 5 +-
.../hadoop/hbase/master/MasterWalManager.java | 42 ++--
.../hbase/master/RegionServerTracker.java | 12 ++-
.../hbase/master/TableNamespaceManager.java | 8 +-
.../master/assignment/AssignmentManager.java| 5 +-
.../master/procedure/DisableTableProcedure.java | 2 +-
.../master/procedure/EnableTableProcedure.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 12 +--
.../hadoop/hbase/TestMetaTableAccessor.java | 16 +++
11 files changed, 177 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6616fd2e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
index dcf6626..c15cfb2 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
@@ -28,6 +28,10 @@ import org.apache.yetus.audience.InterfaceAudience;
public class RetryCounterFactory {
private final RetryConfig retryConfig;
+ public RetryCounterFactory(int sleepIntervalMillis) {
+this(Integer.MAX_VALUE, sleepIntervalMillis);
+ }
+
public RetryCounterFactory(int maxAttempts, int sleepIntervalMillis) {
this(maxAttempts, sleepIntervalMillis, -1);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6616fd2e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index cd1fedf..1a8b8dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -55,6 +55,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -196,6 +197,8 @@ import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.hadoop.hbase.util.RetryCounterFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
@@ -943,11 +946,13 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Start RegionServerTracker with listing of servers found with exiting
SCPs -- these should
// be
hbase git commit: HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can happen if masterprocwals have been cleared).
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 a925a4ce1 -> 39e0b8515
HBASE-21191 Add a holding-pattern if no assign for meta or namespace (Can
happen if masterprocwals have been cleared).
Add a check for hbase:meta being online before we go to read it.
If not online, move into a holding-pattern until rectified, probably
by external operator.
Incorporates bulk of patch made by Allan Yang over on HBASE-21035.
M
hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
Add a Constructor for case where retries are for ever.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Move stuff around so that the first hbase:meta read is the AM#loadMeta.
Previously, checking table state and/or favored nodes could end up
trying to read a meta that was not onlined holding up master startup.
Do similar for the namespace table. Adds new methods isMeta and
isNamespace which check that the regions/tables are online.. if not,
we wait logging with a back-off that assigns need to be run.
Signed-off-by: Allan Yang
Signed-off-by: Duo Zhang
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/39e0b851
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/39e0b851
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/39e0b851
Branch: refs/heads/branch-2.1
Commit: 39e0b8515f9090c9f6f1bc3dd82263e7b6739f6b
Parents: a925a4c
Author: Michael Stack
Authored: Wed Sep 12 10:47:33 2018 -0700
Committer: Michael Stack
Committed: Sun Sep 16 21:12:59 2018 -0700
--
.../hadoop/hbase/util/RetryCounterFactory.java | 4 +
.../org/apache/hadoop/hbase/master/HMaster.java | 104 ++--
.../hadoop/hbase/master/MasterRpcServices.java | 5 +-
.../hadoop/hbase/master/MasterWalManager.java | 42 +--
.../hbase/master/RegionServerTracker.java | 12 +-
.../hbase/master/TableNamespaceManager.java | 8 +-
.../master/assignment/AssignmentManager.java| 5 +-
.../master/procedure/DisableTableProcedure.java | 2 +-
.../master/procedure/EnableTableProcedure.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 12 +-
.../hadoop/hbase/TestMetaTableAccessor.java | 16 +++
.../TestMetaInitIfAllProceduresLost.java| 121 +++
12 files changed, 298 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/39e0b851/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
index dcf6626..c15cfb2 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java
@@ -28,6 +28,10 @@ import org.apache.yetus.audience.InterfaceAudience;
public class RetryCounterFactory {
private final RetryConfig retryConfig;
+ public RetryCounterFactory(int sleepIntervalMillis) {
+this(Integer.MAX_VALUE, sleepIntervalMillis);
+ }
+
public RetryCounterFactory(int maxAttempts, int sleepIntervalMillis) {
this(maxAttempts, sleepIntervalMillis, -1);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/39e0b851/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f3fb989..69c1caa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -55,6 +55,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -191,6 +192,8 @@ import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.hadoop.hbase.util.RetryCounterFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
@@ -882,11 +885,13 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Start RegionServerTracker with
hbase git commit: HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 307578b70 -> c98c6c674
HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c98c6c67
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c98c6c67
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c98c6c67
Branch: refs/heads/branch-2.0
Commit: c98c6c67446d96b48398a7d62aa20953b7a6f21c
Parents: 307578b
Author: Francis Liu
Authored: Sun Aug 12 21:27:03 2018 -0700
Committer: Francis Liu
Committed: Sun Sep 16 18:38:49 2018 -0700
--
.../hadoop/hbase/regionserver/HStore.java | 43 +++-
.../hadoop/hbase/regionserver/HStoreFile.java | 37 +++-
.../hbase/regionserver/StoreFileReader.java | 23 +-
.../TestCleanupCompactedFileOnRegionClose.java | 210 +++
4 files changed, 304 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c98c6c67/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 2227e58..01e4457 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -897,7 +897,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
if (CollectionUtils.isNotEmpty(compactedfiles)) {
-removeCompactedfiles(compactedfiles);
+removeCompactedfiles(compactedfiles, true);
}
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
@@ -2504,6 +2504,11 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* Closes and archives the compacted files under this store
*/
public synchronized void closeAndArchiveCompactedFiles() throws IOException {
+closeAndArchiveCompactedFiles(false);
+ }
+
+ @VisibleForTesting
+ public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing)
throws IOException {
// ensure other threads do not attempt to archive the same files on close()
archiveLock.lock();
try {
@@ -2522,7 +2527,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
lock.readLock().unlock();
}
if (CollectionUtils.isNotEmpty(copyCompactedfiles)) {
-removeCompactedfiles(copyCompactedfiles);
+removeCompactedfiles(copyCompactedfiles, storeClosing);
}
} finally {
archiveLock.unlock();
@@ -2534,7 +2539,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* @param compactedfiles The compacted files in this store that are not
active in reads
* @throws IOException
*/
- private void removeCompactedfiles(Collection compactedfiles)
+ private void removeCompactedfiles(Collection compactedfiles,
boolean storeClosing)
throws IOException {
final List filesToRemove = new
ArrayList<>(compactedfiles.size());
for (final HStoreFile file : compactedfiles) {
@@ -2546,11 +2551,29 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
filesToRemove.add(file);
continue;
}
- if (file.isCompactedAway() && !file.isReferencedInReads()) {
+
+ //Compacted files in the list should always be marked compacted
away. In the event
+ //they're contradicting in order to guarantee data consistency
+ //should we choose one and ignore the other?
+ if (storeClosing && !file.isCompactedAway()) {
+String msg =
+"Region closing but StoreFile is in compacted list but not
compacted away: " +
+file.getPath().getName();
+throw new IllegalStateException(msg);
+ }
+
+ //If store is closing we're ignoring any references to keep things
consistent
+ //and remove compacted storefiles from the region directory
+ if (file.isCompactedAway() && (!file.isReferencedInReads() ||
storeClosing)) {
+if (storeClosing && file.isReferencedInReads()) {
+ LOG.debug("Region closing but StoreFile still has references:
{}",
+ file.getPath().getName());
+}
// Even if deleting fails we need not bother as any new scanners
won't be
// able to use the compacted file as the status is
hbase git commit: HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 842e0c974 -> a925a4ce1
HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a925a4ce
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a925a4ce
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a925a4ce
Branch: refs/heads/branch-2.1
Commit: a925a4ce163dfdde594920f822a9ac20fd182107
Parents: 842e0c9
Author: Francis Liu
Authored: Sun Aug 12 21:27:03 2018 -0700
Committer: Francis Liu
Committed: Sun Sep 16 18:38:03 2018 -0700
--
.../hadoop/hbase/regionserver/HStore.java | 43 +++-
.../hadoop/hbase/regionserver/HStoreFile.java | 37 +++-
.../hbase/regionserver/StoreFileReader.java | 23 +-
.../TestCleanupCompactedFileOnRegionClose.java | 210 +++
4 files changed, 304 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a925a4ce/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 80f1f21..39d0cbb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -924,7 +924,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
if (CollectionUtils.isNotEmpty(compactedfiles)) {
-removeCompactedfiles(compactedfiles);
+removeCompactedfiles(compactedfiles, true);
}
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
@@ -2531,6 +2531,11 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* Closes and archives the compacted files under this store
*/
public synchronized void closeAndArchiveCompactedFiles() throws IOException {
+closeAndArchiveCompactedFiles(false);
+ }
+
+ @VisibleForTesting
+ public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing)
throws IOException {
// ensure other threads do not attempt to archive the same files on close()
archiveLock.lock();
try {
@@ -2549,7 +2554,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
lock.readLock().unlock();
}
if (CollectionUtils.isNotEmpty(copyCompactedfiles)) {
-removeCompactedfiles(copyCompactedfiles);
+removeCompactedfiles(copyCompactedfiles, storeClosing);
}
} finally {
archiveLock.unlock();
@@ -2561,7 +2566,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* @param compactedfiles The compacted files in this store that are not
active in reads
* @throws IOException
*/
- private void removeCompactedfiles(Collection compactedfiles)
+ private void removeCompactedfiles(Collection compactedfiles,
boolean storeClosing)
throws IOException {
final List filesToRemove = new
ArrayList<>(compactedfiles.size());
for (final HStoreFile file : compactedfiles) {
@@ -2573,11 +2578,29 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
filesToRemove.add(file);
continue;
}
- if (file.isCompactedAway() && !file.isReferencedInReads()) {
+
+ //Compacted files in the list should always be marked compacted
away. In the event
+ //they're contradicting in order to guarantee data consistency
+ //should we choose one and ignore the other?
+ if (storeClosing && !file.isCompactedAway()) {
+String msg =
+"Region closing but StoreFile is in compacted list but not
compacted away: " +
+file.getPath().getName();
+throw new IllegalStateException(msg);
+ }
+
+ //If store is closing we're ignoring any references to keep things
consistent
+ //and remove compacted storefiles from the region directory
+ if (file.isCompactedAway() && (!file.isReferencedInReads() ||
storeClosing)) {
+if (storeClosing && file.isReferencedInReads()) {
+ LOG.debug("Region closing but StoreFile still has references:
{}",
+ file.getPath().getName());
+}
// Even if deleting fails we need not bother as any new scanners
won't be
// able to use the compacted file as the status is
hbase git commit: HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Repository: hbase
Updated Branches:
refs/heads/branch-2 966e3751c -> 7a1b4d449
HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7a1b4d44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7a1b4d44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7a1b4d44
Branch: refs/heads/branch-2
Commit: 7a1b4d449df10dbee2f92acdfa5865cf9d9283e2
Parents: 966e375
Author: Francis Liu
Authored: Sun Aug 12 21:27:03 2018 -0700
Committer: Francis Liu
Committed: Sun Sep 16 18:28:24 2018 -0700
--
.../hadoop/hbase/regionserver/HStore.java | 43 +++-
.../hadoop/hbase/regionserver/HStoreFile.java | 37 +++-
.../hbase/regionserver/StoreFileReader.java | 23 +-
.../TestCleanupCompactedFileOnRegionClose.java | 210 +++
4 files changed, 304 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7a1b4d44/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 314547e..9a381f6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -924,7 +924,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
if (CollectionUtils.isNotEmpty(compactedfiles)) {
-removeCompactedfiles(compactedfiles);
+removeCompactedfiles(compactedfiles, true);
}
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
@@ -2533,6 +2533,11 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* Closes and archives the compacted files under this store
*/
public synchronized void closeAndArchiveCompactedFiles() throws IOException {
+closeAndArchiveCompactedFiles(false);
+ }
+
+ @VisibleForTesting
+ public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing)
throws IOException {
// ensure other threads do not attempt to archive the same files on close()
archiveLock.lock();
try {
@@ -2551,7 +2556,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
lock.readLock().unlock();
}
if (CollectionUtils.isNotEmpty(copyCompactedfiles)) {
-removeCompactedfiles(copyCompactedfiles);
+removeCompactedfiles(copyCompactedfiles, storeClosing);
}
} finally {
archiveLock.unlock();
@@ -2563,7 +2568,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* @param compactedfiles The compacted files in this store that are not
active in reads
* @throws IOException
*/
- private void removeCompactedfiles(Collection compactedfiles)
+ private void removeCompactedfiles(Collection compactedfiles,
boolean storeClosing)
throws IOException {
final List filesToRemove = new
ArrayList<>(compactedfiles.size());
for (final HStoreFile file : compactedfiles) {
@@ -2575,11 +2580,29 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
filesToRemove.add(file);
continue;
}
- if (file.isCompactedAway() && !file.isReferencedInReads()) {
+
+ //Compacted files in the list should always be marked compacted
away. In the event
+ //they're contradicting in order to guarantee data consistency
+ //should we choose one and ignore the other?
+ if (storeClosing && !file.isCompactedAway()) {
+String msg =
+"Region closing but StoreFile is in compacted list but not
compacted away: " +
+file.getPath().getName();
+throw new IllegalStateException(msg);
+ }
+
+ //If store is closing we're ignoring any references to keep things
consistent
+ //and remove compacted storefiles from the region directory
+ if (file.isCompactedAway() && (!file.isReferencedInReads() ||
storeClosing)) {
+if (storeClosing && file.isReferencedInReads()) {
+ LOG.debug("Region closing but StoreFile still has references:
{}",
+ file.getPath().getName());
+}
// Even if deleting fails we need not bother as any new scanners
won't be
// able to use the compacted file as the status is already
hbase git commit: HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Repository: hbase
Updated Branches:
refs/heads/master e86c73602 -> cdfe80889
HBASE-20704 Sometimes some compacted storefiles are not archived on region close
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cdfe8088
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cdfe8088
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cdfe8088
Branch: refs/heads/master
Commit: cdfe80889259e7e914ddb541962184315228d32e
Parents: e86c736
Author: Francis Liu
Authored: Sun Aug 12 21:27:03 2018 -0700
Committer: Francis Liu
Committed: Sun Sep 16 18:26:28 2018 -0700
--
.../hadoop/hbase/regionserver/HStore.java | 43 +++-
.../hadoop/hbase/regionserver/HStoreFile.java | 37 +++-
.../hbase/regionserver/StoreFileReader.java | 23 +-
.../TestCleanupCompactedFileOnRegionClose.java | 210 +++
4 files changed, 304 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cdfe8088/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 418eecc..717a466 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -927,7 +927,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
if (CollectionUtils.isNotEmpty(compactedfiles)) {
-removeCompactedfiles(compactedfiles);
+removeCompactedfiles(compactedfiles, true);
}
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
@@ -2575,6 +2575,11 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* Closes and archives the compacted files under this store
*/
public synchronized void closeAndArchiveCompactedFiles() throws IOException {
+closeAndArchiveCompactedFiles(false);
+ }
+
+ @VisibleForTesting
+ public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing)
throws IOException {
// ensure other threads do not attempt to archive the same files on close()
archiveLock.lock();
try {
@@ -2593,7 +2598,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
lock.readLock().unlock();
}
if (CollectionUtils.isNotEmpty(copyCompactedfiles)) {
-removeCompactedfiles(copyCompactedfiles);
+removeCompactedfiles(copyCompactedfiles, storeClosing);
}
} finally {
archiveLock.unlock();
@@ -2604,7 +2609,7 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
* Archives and removes the compacted files
* @param compactedfiles The compacted files in this store that are not
active in reads
*/
- private void removeCompactedfiles(Collection compactedfiles)
+ private void removeCompactedfiles(Collection compactedfiles,
boolean storeClosing)
throws IOException {
final List filesToRemove = new
ArrayList<>(compactedfiles.size());
final List storeFileSizes = new ArrayList<>(compactedfiles.size());
@@ -2622,13 +2627,31 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
storeFileSizes.add(length);
continue;
}
- if (file.isCompactedAway() && !file.isReferencedInReads()) {
+
+ //Compacted files in the list should always be marked compacted
away. In the event
+ //they're contradicting in order to guarantee data consistency
+ //should we choose one and ignore the other?
+ if (storeClosing && !file.isCompactedAway()) {
+String msg =
+"Region closing but StoreFile is in compacted list but not
compacted away: " +
+file.getPath().getName();
+throw new IllegalStateException(msg);
+ }
+
+ //If store is closing we're ignoring any references to keep things
consistent
+ //and remove compacted storefiles from the region directory
+ if (file.isCompactedAway() && (!file.isReferencedInReads() ||
storeClosing)) {
+if (storeClosing && file.isReferencedInReads()) {
+ LOG.debug("Region closing but StoreFile still has references:
{}",
+ file.getPath().getName());
+}
// Even if deleting fails we need not bother as any new scanners
won't be
// able to use the
hbase-site git commit: Published site at e86c7360288cc077cd23af495fb443179ed0157b.
Repository: hbase-site Updated Branches: refs/heads/asf-site 49642583e -> 28cde851e Published site at e86c7360288cc077cd23af495fb443179ed0157b. Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/28cde851 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/28cde851 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/28cde851 Branch: refs/heads/asf-site Commit: 28cde851e0e1ea743a41183cd5408aacf84a24e9 Parents: 4964258 Author: jenkins Authored: Sun Sep 16 14:51:50 2018 + Committer: jenkins Committed: Sun Sep 16 14:51:50 2018 + -- acid-semantics.html | 4 +-- apache_hbase_reference_guide.pdf| 4 +-- book.html | 2 +- bulk-loads.html | 4 +-- checkstyle-aggregate.html | 10 +++--- coc.html| 4 +-- dependencies.html | 4 +-- dependency-convergence.html | 4 +-- dependency-info.html| 4 +-- dependency-management.html | 4 +-- devapidocs/constant-values.html | 4 +-- .../org/apache/hadoop/hbase/Version.html| 4 +-- downloads.html | 32 ++-- export_control.html | 4 +-- index.html | 4 +-- integration.html| 4 +-- issue-tracking.html | 4 +-- license.html| 4 +-- mail-lists.html | 4 +-- metrics.html| 4 +-- old_news.html | 4 +-- plugin-management.html | 4 +-- plugins.html| 4 +-- poweredbyhbase.html | 4 +-- project-info.html | 4 +-- project-reports.html| 4 +-- project-summary.html| 4 +-- pseudo-distributed.html | 4 +-- replication.html| 4 +-- resources.html | 4 +-- source-repository.html | 4 +-- sponsors.html | 4 +-- supportingprojects.html | 4 +-- team-list.html | 4 +-- 34 files changed, 98 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase-site/blob/28cde851/acid-semantics.html -- diff --git a/acid-semantics.html b/acid-semantics.html index 0724701..a6c46a0 100644 --- a/acid-semantics.html +++ b/acid-semantics.html @@ -7,7 +7,7 @@ - + Apache HBase Apache HBase (TM) ACID Properties @@ -601,7 +601,7 @@ under the License. --> https://www.apache.org/;>The Apache Software Foundation. All rights reserved. - Last Published: 2018-09-15 + Last Published: 2018-09-16 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/28cde851/apache_hbase_reference_guide.pdf -- diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf index 8b92111..e1c14d0 100644 --- a/apache_hbase_reference_guide.pdf +++ b/apache_hbase_reference_guide.pdf @@ -5,8 +5,8 @@ /Author (Apache HBase Team) /Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2) /Producer (Apache HBase Team) -/ModDate (D:20180915143204+00'00') -/CreationDate (D:20180915144754+00'00') +/ModDate (D:20180916143310+00'00') +/CreationDate (D:20180916144831+00'00') >> endobj 2 0 obj http://git-wip-us.apache.org/repos/asf/hbase-site/blob/28cde851/book.html -- diff --git a/book.html b/book.html index de4e74d..acf5232 100644 --- a/book.html +++ b/book.html @@ -41284,7 +41284,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/ Version 3.0.0-SNAPSHOT -Last updated 2018-09-15 14:32:04 UTC +Last updated 2018-09-16 14:33:10 UTC http://git-wip-us.apache.org/repos/asf/hbase-site/blob/28cde851/bulk-loads.html -- diff --git a/bulk-loads.html b/bulk-loads.html index c5df2be..9a7c919 100644 --- a/bulk-loads.html +++ b/bulk-loads.html @@ -7,7 +7,7 @@ - +
hbase-site git commit: INFRA-10751 Empty commit
Repository: hbase-site Updated Branches: refs/heads/asf-site 28cde851e -> bcb894667 INFRA-10751 Empty commit Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/bcb89466 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/bcb89466 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/bcb89466 Branch: refs/heads/asf-site Commit: bcb894667bf8c3a1f7d6e6dbc1da2f9938d1446b Parents: 28cde85 Author: jenkins Authored: Sun Sep 16 14:52:01 2018 + Committer: jenkins Committed: Sun Sep 16 14:52:01 2018 + -- --
