Author: stack
Date: Fri Jul 24 22:42:43 2009
New Revision: 797682
URL: http://svn.apache.org/viewvc?rev=797682&view=rev
Log:
HBASE-1683 OOME on master splitting logs; stuck, won't go down
Modified:
hadoop/hbase/branches/0.19/CHANGES.txt
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=797682&r1=797681&r2=797682&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Fri Jul 24 22:42:43 2009
@@ -28,6 +28,7 @@
Rather, just let client fail
HBASE-1651 client is broken, it requests ROOT region location from ZK
too much
+ HBASE-1683 OOME on master splitting logs; stuck, won't go down
Release 0.19.3 - May 27th, 2009
BUG FIXES
Modified:
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HConstants.java?rev=797682&r1=797681&r2=797682&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HConstants.java
(original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HConstants.java
Fri Jul 24 22:42:43 2009
@@ -120,14 +120,6 @@
/** Default size of a reservation block */
static final int DEFAULT_SIZE_RESERVATION_BLOCK = 1024 * 1024 * 5;
- /** Default number of threads to use when log splitting
- * to rewrite the logs. More means faster but bigger mem consumption */
- static final int DEFAULT_NUMBER_LOG_WRITER_THREAD = 10;
-
- /** Default number of logs to read concurrently
- * when log splitting. More means faster but bigger mem consumption */
- static final int DEFAULT_NUMBER_CONCURRENT_LOG_READS = 10;
-
// Always store the location of the root table's HRegion.
// This HRegion is never split.
@@ -280,4 +272,4 @@
public static final int MODIFY_TABLE_FLUSH = MODIFY_CLOSE_REGION + 1;
public static final int MODIFY_TABLE_MAJOR_COMPACT = MODIFY_TABLE_FLUSH + 1;
-}
\ No newline at end of file
+}
Modified:
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=797682&r1=797681&r2=797682&view=diff
==============================================================================
---
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
(original)
+++
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Fri Jul 24 22:42:43 2009
@@ -780,16 +780,26 @@
final Map<byte [], WriterAndPath> logWriters =
new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR);
List<Path> splits = null;
+
+ // Number of threads to use when log splitting to rewrite the logs.
+ // More means faster but bigger mem consumption.
+ int logWriterThreads =
+ conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);
+
+ // Number of logs to read concurrently when log splitting.
+ // More means faster but bigger mem consumption */
+ int concurrentLogReads =
+ conf.getInt("hbase.regionserver.hlog.splitlog.reader.threads", 3);
+
try {
int maxSteps = Double.valueOf(Math.ceil((logfiles.length * 1.0) /
- DEFAULT_NUMBER_CONCURRENT_LOG_READS)).intValue();
+ concurrentLogReads)).intValue();
for(int step = 0; step < maxSteps; step++) {
final Map<byte[], LinkedList<HLogEntry>> logEntries =
new TreeMap<byte[], LinkedList<HLogEntry>>(Bytes.BYTES_COMPARATOR);
// Stop at logfiles.length when it's the last step
int endIndex = step == maxSteps - 1 ? logfiles.length :
- step*DEFAULT_NUMBER_CONCURRENT_LOG_READS +
- DEFAULT_NUMBER_CONCURRENT_LOG_READS;
+ step*concurrentLogReads + concurrentLogReads;
for (int i = (step * 10); i < endIndex; i++) {
if (LOG.isDebugEnabled()) {
LOG.debug("Splitting hlog " + (i + 1) + " of " + logfiles.length
+
@@ -854,7 +864,7 @@
}
}
ExecutorService threadPool =
- Executors.newFixedThreadPool(DEFAULT_NUMBER_LOG_WRITER_THREAD);
+ Executors.newFixedThreadPool(logWriterThreads);
for (final byte[] key : logEntries.keySet()) {
Thread thread = new Thread(Bytes.toString(key)) {
