Author: nspiegelberg
Date: Fri Jun 10 23:32:42 2011
New Revision: 1134481
URL: http://svn.apache.org/viewvc?rev=1134481&view=rev
Log:
HBASE-3794 : Ability to Discard Bad HTable Puts
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HTable.java
Modified: hbase/branches/0.90/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1134481&r1=1134480&r2=1134481&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Fri Jun 10 23:32:42 2011
@@ -29,6 +29,7 @@ Release 0.90.4 - Unreleased
HBASE-3976 Disable block cache on compactions (Karthik Sankarachary)
HBASE-3892 Table can't disable (Gao Jinchao)
HBASE-3894 Thread contention over row locks set monitor (Dave Latham)
+ HBASE-3794 Ability to Discard Bad HTable Puts
IMPROVEMENT
HBASE-3882 hbase-config.sh needs to be updated so it can auto-detects the
Modified:
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1134481&r1=1134480&r2=1134481&view=diff
==============================================================================
---
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HTable.java
(original)
+++
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HTable.java
Fri Jun 10 23:32:42 2011
@@ -93,6 +93,7 @@ public class HTable implements HTableInt
private volatile Configuration configuration;
private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
private long writeBufferSize;
+ private boolean clearBufferOnFail;
private boolean autoFlush;
private long currentWriteBufferSize;
protected int scannerCaching;
@@ -171,6 +172,7 @@ public class HTable implements HTableInt
this.configuration = conf;
this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
+ this.clearBufferOnFail = true;
this.autoFlush = true;
this.currentWriteBufferSize = 0;
this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
@@ -825,10 +827,15 @@ public class HTable implements HTableInt
try {
connection.processBatchOfPuts(writeBuffer, tableName, pool);
} finally {
- // the write buffer was adjusted by processBatchOfPuts
- currentWriteBufferSize = 0;
- for (Put aPut : writeBuffer) {
- currentWriteBufferSize += aPut.heapSize();
+ if (clearBufferOnFail) {
+ writeBuffer.clear();
+ currentWriteBufferSize = 0;
+ } else {
+ // the write buffer was adjusted by processBatchOfPuts
+ currentWriteBufferSize = 0;
+ for (Put aPut : writeBuffer) {
+ currentWriteBufferSize += aPut.heapSize();
+ }
}
}
}
@@ -897,25 +904,45 @@ public class HTable implements HTableInt
}
/**
+ * See {@link #setAutoFlush(boolean, boolean)}
+ *
+ * @param autoFlush
+ * Whether or not to enable 'auto-flush'.
+ */
+ public void setAutoFlush(boolean autoFlush) {
+ setAutoFlush(autoFlush, autoFlush);
+ }
+
+ /**
* Turns 'auto-flush' on or off.
* <p>
* When enabled (default), {@link Put} operations don't get buffered/delayed
- * and are immediately executed. This is slower but safer.
+ * and are immediately executed. Failed operations are not retried. This is
+ * slower but safer.
* <p>
- * Turning this off means that multiple {@link Put}s will be accepted before
- * any RPC is actually sent to do the write operations. If the application
- * dies before pending writes get flushed to HBase, data will be lost.
- * Other side effects may include the fact that the application thinks a
- * {@link Put} was executed successfully whereas it was in fact only
- * buffered and the operation may fail when attempting to flush all pending
- * writes. In that case though, the code will retry the failed {@link Put}
- * upon its next attempt to flush the buffer.
+ * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
+ * accepted before any RPC is actually sent to do the write operations. If
the
+ * application dies before pending writes get flushed to HBase, data will be
+ * lost.
+ * <p>
+ * When you turn {@link #autoFlush} off, you should also consider the
+ * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put)
+ * requests will be retried on failure until successful. However, this can
+ * pollute the writeBuffer and slow down batching performance. Additionally,
+ * you may want to issue a number of Put requests and call
+ * {@link #flushCommits()} as a barrier. In both use cases, consider setting
+ * clearBufferOnFail to true to erase the buffer after {@link
#flushCommits()}
+ * has been called, regardless of success.
*
- * @param autoFlush Whether or not to enable 'auto-flush'.
+ * @param autoFlush
+ * Whether or not to enable 'auto-flush'.
+ * @param clearBufferOnFail
+ * Whether to keep Put failures in the writeBuffer
* @see #flushCommits
*/
- public void setAutoFlush(boolean autoFlush) {
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
this.autoFlush = autoFlush;
+ this.clearBufferOnFail = autoFlush || clearBufferOnFail;
}
/**
