Author: todd
Date: Tue Jun 15 22:44:07 2010
New Revision: 955077
URL: http://svn.apache.org/viewvc?rev=955077&view=rev
Log:
HBASE-2733. Replacement of LATEST_TIMESTAMP with real timestamp was broken by
HBASE-2353
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=955077&r1=955076&r2=955077&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue Jun 15 22:44:07 2010
@@ -388,6 +388,8 @@ Release 0.21.0 - Unreleased
HBASE-2732 TestZooKeeper was broken, HBASE-2691 showed it
HBASE-2670 Provide atomicity for readers even when new insert has
same timestamp as current row.
+ HBASE-2733 Replacement of LATEST_TIMESTAMP with real timestamp was broken
+ by HBASE-2353.
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=955077&r1=955076&r2=955077&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Jun 15 22:44:07 2010
@@ -1327,7 +1327,7 @@ public class HRegion implements HeapSize
// bunch up all edits across all column families into a
// single WALEdit.
WALEdit walEdit = new WALEdit();
- addFamilyMapToWALEdit(familyMap, byteNow, walEdit);
+ addFamilyMapToWALEdit(familyMap, walEdit);
this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
walEdit, now);
}
@@ -1520,9 +1520,18 @@ public class HRegion implements HeapSize
}
// We've now grabbed as many puts off the list as we can
assert numReadyToWrite > 0;
+
+ // ------------------------------------
+ // STEP 2. Update any LATEST_TIMESTAMP timestamps
+ // ----------------------------------
+ for (int i = firstIndex; i < lastIndexExclusive; i++) {
+ updateKVTimestamps(
+ batchOp.operations[i].getFirst().getFamilyMap().values(),
+ byteNow);
+ }
// ------------------------------------
- // STEP 2. Write to WAL
+ // STEP 3. Write to WAL
// ----------------------------------
WALEdit walEdit = new WALEdit();
for (int i = firstIndex; i < lastIndexExclusive; i++) {
@@ -1531,7 +1540,7 @@ public class HRegion implements HeapSize
Put p = batchOp.operations[i].getFirst();
if (!p.getWriteToWAL()) continue;
- addFamilyMapToWALEdit(p.getFamilyMap(), byteNow, walEdit);
+ addFamilyMapToWALEdit(p.getFamilyMap(), walEdit);
}
// Append the edit to WAL
@@ -1539,7 +1548,7 @@ public class HRegion implements HeapSize
walEdit, now);
// ------------------------------------
- // STEP 3. Write back to memstore
+ // STEP 4. Write back to memstore
// ----------------------------------
long addedSize = 0;
for (int i = firstIndex; i < lastIndexExclusive; i++) {
@@ -1636,21 +1645,17 @@ public class HRegion implements HeapSize
/**
- * Checks if any stamps is Long.MAX_VALUE. If so, sets them to now.
- * <p>
- * This acts to replace {...@link HConstants#LATEST_TIMESTAMP} with
{...@code now}.
- * @param keys
- * @param now
- * @return <code>true</code> when updating the time stamp completed.
+ * Replaces any KV timestamps set to {...@link HConstants#LATEST_TIMESTAMP}
+ * with the provided current timestamp.
*/
- private boolean updateKeys(final List<KeyValue> keys, final byte[] now) {
- if (keys == null || keys.isEmpty()) {
- return false;
- }
- for (KeyValue key : keys) {
- key.updateLatestStamp(now);
+ private void updateKVTimestamps(
+ final Iterable<List<KeyValue>> keyLists, final byte[] now) {
+ for (List<KeyValue> keys: keyLists) {
+ if (keys == null) continue;
+ for (KeyValue key : keys) {
+ key.updateLatestStamp(now);
+ }
}
- return true;
}
// /*
@@ -1754,7 +1759,7 @@ public class HRegion implements HeapSize
this.updatesLock.readLock().lock();
try {
checkFamilies(familyMap.keySet());
-
+ updateKVTimestamps(familyMap.values(), byteNow);
// write/sync to WAL should happen before we touch memstore.
//
// If order is reversed, i.e. we write to memstore first, and
@@ -1762,7 +1767,7 @@ public class HRegion implements HeapSize
// will contain uncommitted transactions.
if (writeToWAL) {
WALEdit walEdit = new WALEdit();
- addFamilyMapToWALEdit(familyMap, byteNow, walEdit);
+ addFamilyMapToWALEdit(familyMap, walEdit);
this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
walEdit, now);
}
@@ -1822,22 +1827,15 @@ public class HRegion implements HeapSize
/**
* Append the given map of family->edits to a WALEdit data structure.
- * Also updates the timestamps of the edits where they have not
- * been specified by the user. This does not write to the HLog itself.
+ * This does not write to the HLog itself.
* @param familyMap map of family->edits
- * @param byteNow timestamp to use when unspecified
* @param walEdit the destination entry to append into
*/
private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
- byte[] byteNow, WALEdit walEdit) {
+ WALEdit walEdit) {
for (List<KeyValue> edits : familyMap.values()) {
- // update timestamp on keys if required.
- if (updateKeys(edits, byteNow)) {
- // bunch up all edits across all column families into a
- // single WALEdit.
- for (KeyValue kv : edits) {
- walEdit.add(kv);
- }
+ for (KeyValue kv : edits) {
+ walEdit.add(kv);
}
}
}
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=955077&r1=955076&r2=955077&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Tue Jun 15 22:44:07 2010
@@ -345,6 +345,7 @@ public class TestHRegion extends HBaseTe
byte[] val = Bytes.toBytes("val");
initHRegion(b, getName(), cf);
+ HLog.getSyncOps(); // clear counter from prior tests
assertEquals(0, HLog.getSyncOps());
LOG.info("First a batch put with all valid puts");
@@ -833,6 +834,53 @@ public class TestHRegion extends HBaseTe
result = region.get(get, null);
assertEquals(0, result.size());
}
+
+ /**
+ * Tests that the special LATEST_TIMESTAMP option for puts gets
+ * replaced by the actual timestamp
+ */
+ public void testPutWithLatestTS() throws IOException {
+ byte [] tableName = Bytes.toBytes("testtable");
+ byte [] fam = Bytes.toBytes("info");
+ byte [][] families = {fam};
+ String method = this.getName();
+ initHRegion(tableName, method, families);
+
+ byte [] row = Bytes.toBytes("row1");
+ // column names
+ byte [] qual = Bytes.toBytes("qual");
+
+ // add data with LATEST_TIMESTAMP, put without WAL
+ Put put = new Put(row);
+ put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
+ region.put(put, false);
+
+ // Make sure it shows up with an actual timestamp
+ Get get = new Get(row).addColumn(fam, qual);
+ Result result = region.get(get, null);
+ assertEquals(1, result.size());
+ KeyValue kv = result.raw()[0];
+ LOG.info("Got: " + kv);
+ assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
+ kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
+
+ // Check same with WAL enabled (historically these took different
+ // code paths, so check both)
+ row = Bytes.toBytes("row2");
+ put = new Put(row);
+ put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
+ region.put(put, true);
+
+ // Make sure it shows up with an actual timestamp
+ get = new Get(row).addColumn(fam, qual);
+ result = region.get(get, null);
+ assertEquals(1, result.size());
+ kv = result.raw()[0];
+ LOG.info("Got: " + kv);
+ assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
+ kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
+
+ }
public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
byte [] tableName = Bytes.toBytes("test_table");
