Author: stack
Date: Thu Nov 5 21:50:35 2009
New Revision: 833194
URL: http://svn.apache.org/viewvc?rev=833194&view=rev
Log:
HBASE-1949 KeyValue expiration by Time-to-Live during major compaction is broken
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Nov 5 21:50:35 2009
@@ -94,6 +94,8 @@
when deleting a lot of values
HBASE-1781 Weird behavior of WildcardColumnTracker.checkColumn(),
looks like recursive loop
+ HBASE-1949 KeyValue expiration by Time-to-Live during major compaction is
+ broken (Gary Helmling via Stack)
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
Thu Nov 5 21:50:35 2009
@@ -231,8 +231,10 @@
*/
long timestamp = Bytes.toLong(bytes, offset);
if(isExpired(timestamp)) {
- // reached the expired part, for scans, this indicates we're done.
- return MatchCode.NEXT; // done_row
+ /* KeyValue is expired, skip but don't early out since a non-expired
+ * kv could come next.
+ */
+ return MatchCode.SKIP; // go to next kv
}
offset += Bytes.SIZEOF_LONG;
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Thu Nov 5 21:50:35 2009
@@ -122,9 +122,8 @@
long timestamp = kv.getTimestamp();
if (isExpired(timestamp)) {
- // done, the rest wil also be expired as well.
- stickyNextRow = true;
- return MatchCode.SEEK_NEXT_ROW;
+ // done, the rest of this column will also be expired as well.
+ return MatchCode.SEEK_NEXT_COL;
}
byte type = kv.getType();
Modified:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
(original)
+++
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
Thu Nov 5 21:50:35 2009
@@ -27,6 +27,7 @@
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
import org.apache.hadoop.hbase.KeyValue.KeyComparator;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
@@ -162,4 +163,103 @@
}
}
+
+ /**
+ * Verify that {...@link QueryMatcher} only skips expired KeyValue
+ * instances and does not exit early from the row (skipping
+ * later non-expired KeyValues). This version mimics a Get with
+ * explicitly specified column qualifiers.
+ *
+ * @throws IOException
+ */
+ public void testMatch_ExpiredExplicit()
+ throws IOException {
+
+ long testTTL = 1000;
+ MatchCode [] expected = new MatchCode[] {
+ MatchCode.SKIP,
+ MatchCode.INCLUDE,
+ MatchCode.SKIP,
+ MatchCode.INCLUDE,
+ MatchCode.SKIP,
+ MatchCode.NEXT
+ };
+
+ QueryMatcher qm = new QueryMatcher(get, fam2,
+ get.getFamilyMap().get(fam2), testTTL, rowComparator, 1);
+
+ long now = System.currentTimeMillis();
+ KeyValue [] kvs = new KeyValue[] {
+ new KeyValue(row1, fam2, col1, now-100, data),
+ new KeyValue(row1, fam2, col2, now-50, data),
+ new KeyValue(row1, fam2, col3, now-5000, data),
+ new KeyValue(row1, fam2, col4, now-500, data),
+ new KeyValue(row1, fam2, col5, now-10000, data),
+ new KeyValue(row2, fam1, col1, now-10, data)
+ };
+
+ List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
+ for (KeyValue kv : kvs) {
+ actual.add( qm.match(kv) );
+ }
+
+ assertEquals(expected.length, actual.size());
+ for (int i=0; i<expected.length; i++) {
+ if(PRINT){
+ System.out.println("expected "+expected[i]+
+ ", actual " +actual.get(i));
+ }
+ assertEquals(expected[i], actual.get(i));
+ }
+ }
+
+
+ /**
+ * Verify that {...@link QueryMatcher} only skips expired KeyValue
+ * instances and does not exit early from the row (skipping
+ * later non-expired KeyValues). This version mimics a Get with
+ * wildcard-inferred column qualifiers.
+ *
+ * @throws IOException
+ */
+ public void testMatch_ExpiredWildcard()
+ throws IOException {
+
+ long testTTL = 1000;
+ MatchCode [] expected = new MatchCode[] {
+ MatchCode.INCLUDE,
+ MatchCode.INCLUDE,
+ MatchCode.SKIP,
+ MatchCode.INCLUDE,
+ MatchCode.SKIP,
+ MatchCode.NEXT
+ };
+
+ QueryMatcher qm = new QueryMatcher(get, fam2,
+ null, testTTL, rowComparator, 1);
+
+ long now = System.currentTimeMillis();
+ KeyValue [] kvs = new KeyValue[] {
+ new KeyValue(row1, fam2, col1, now-100, data),
+ new KeyValue(row1, fam2, col2, now-50, data),
+ new KeyValue(row1, fam2, col3, now-5000, data),
+ new KeyValue(row1, fam2, col4, now-500, data),
+ new KeyValue(row1, fam2, col5, now-10000, data),
+ new KeyValue(row2, fam1, col1, now-10, data)
+ };
+
+ List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
+ for (KeyValue kv : kvs) {
+ actual.add( qm.match(kv) );
+ }
+
+ assertEquals(expected.length, actual.size());
+ for (int i=0; i<expected.length; i++) {
+ if(PRINT){
+ System.out.println("expected "+expected[i]+
+ ", actual " +actual.get(i));
+ }
+ assertEquals(expected[i], actual.get(i));
+ }
+ }
}
Modified:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
(original)
+++
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Thu Nov 5 21:50:35 2009
@@ -354,4 +354,46 @@
results.clear();
assertEquals(false, scan.next(results));
}
+
+ /**
+ * Test expiration of KeyValues in combination with a configured TTL for
+ * a column family (as should be triggered in a major compaction).
+ */
+ public void testWildCardTtlScan() throws IOException {
+ long now = System.currentTimeMillis();
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put,
"dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ Scan scan = new Scan();
+ scan.setMaxVersions(1);
+ StoreScanner scanner =
+ new StoreScanner(scan, CF, 500, KeyValue.COMPARATOR,
+ null, scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scanner.next(results));
+ assertEquals(2, results.size());
+ assertEquals(kvs[1], results.get(0));
+ assertEquals(kvs[2], results.get(1));
+ results.clear();
+
+ assertEquals(true, scanner.next(results));
+ assertEquals(3, results.size());
+ assertEquals(kvs[4], results.get(0));
+ assertEquals(kvs[5], results.get(1));
+ assertEquals(kvs[6], results.get(2));
+ results.clear();
+
+ assertEquals(false, scanner.next(results));
+ }
}
