Repository: hbase Updated Branches: refs/heads/branch-1 2d83d5545 -> f8eb1962d
HBASE-11728 - Data loss while scanning using PREFIX_TREE DATA-BLOCK-ENCODING (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8eb1962 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8eb1962 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8eb1962 Branch: refs/heads/branch-1 Commit: f8eb1962dc9e92122d00cccfede819014a1cc8f6 Parents: 2d83d55 Author: Ramkrishna <[email protected]> Authored: Mon Aug 18 22:34:26 2014 +0530 Committer: Ramkrishna <[email protected]> Committed: Mon Aug 18 22:34:26 2014 +0530 ---------------------------------------------------------------------- .../codec/prefixtree/PrefixTreeSeeker.java | 9 +- .../decode/PrefixTreeArrayScanner.java | 23 ++- .../hbase/io/encoding/TestPrefixTree.java | 180 +++++++++++++++++++ 3 files changed, 209 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f8eb1962/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java ---------------------------------------------------------------------- diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index 65dd1ce..1184af0 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -45,6 +45,7 @@ public class PrefixTreeSeeker implements EncodedSeeker { protected ByteBuffer block; protected boolean includeMvccVersion; protected PrefixTreeArraySearcher ptSearcher; + protected boolean movedToPrevious = false; public PrefixTreeSeeker(boolean includeMvccVersion) { this.includeMvccVersion = includeMvccVersion; @@ -119,7 +120,11 @@ public class PrefixTreeSeeker implements EncodedSeeker { @Override public boolean next() { - return ptSearcher.advance(); + boolean advance = ptSearcher.advance(); + if (ptSearcher.hasMovedToPreviousAsPartOfSeek()) { + ptSearcher.setMovedToPreviousAsPartOfSeek(false); + } + return advance; } // @Override @@ -207,6 +212,7 @@ public class PrefixTreeSeeker implements EncodedSeeker { if (CellScannerPosition.AT == position) { if (seekBefore) { + // We need not set movedToPrevious because the intention is to seekBefore ptSearcher.previous(); return 1; } @@ -217,6 +223,7 @@ public class PrefixTreeSeeker implements EncodedSeeker { if (CellScannerPosition.AFTER == position) { if (!ptSearcher.isBeforeFirst()) { ptSearcher.previous(); + ptSearcher.setMovedToPreviousAsPartOfSeek(true); } return 1; } http://git-wip-us.apache.org/repos/asf/hbase/blob/f8eb1962/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java ---------------------------------------------------------------------- diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java index ef38f2d..5e8c2ec 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java @@ -60,6 +60,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne protected boolean nubCellsRemain; protected int currentCellIndex; + protected boolean movedToPrevious; /*********************** construct ******************************/ @@ -255,11 +256,17 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne if (currentRowNode.hasMoreFanNodes()) { followNextFan(); if (currentRowNode.hasOccurrences()) { + // found some values currentCellIndex = 0; return true; - }// found some values + } } else { - discardCurrentRowNode(true); + if (movedToPrevious && currentRowNode.hasOccurrences() + && currentRowNode.getFanIndex() == getNextFanIndex()) { + followFan(getNextFanIndex()); + } else { + discardCurrentRowNode(true); + } } } return false;// went past the end @@ -296,6 +303,10 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne followFan(currentFanPosition); } + protected int getNextFanIndex() { + return rowNodes[rowNodeStackIndex + 1].getFanIndex(); + } + protected void followNextFan() { int nextFanPosition = currentRowNode.getFanIndex() + 1; followFan(nextFanPosition); @@ -527,4 +538,12 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne return tagsBuffer.length; } + public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) { + this.movedToPrevious = movedToPrevious; + } + + public boolean hasMovedToPreviousAsPartOfSeek() { + return this.movedToPrevious; + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/f8eb1962/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java new file mode 100644 index 0000000..c9be776 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.encoding; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category(MediumTests.class) +public class TestPrefixTree { + private static final String row4 = "a-b-B-2-1402397300-1402416535"; + private static final byte[] row4_bytes = Bytes.toBytes(row4); + private static final String row3 = "a-b-A-1-1402397227-1402415999"; + private static final byte[] row3_bytes = Bytes.toBytes(row3); + private static final String row2 = "a-b-A-1-1402329600-1402396277"; + private static final byte[] row2_bytes = Bytes.toBytes(row2); + private static final String row1 = "a-b-A-1"; + private static final byte[] row1_bytes = Bytes.toBytes(row1); + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static byte[] fam = Bytes.toBytes("cf_1"); + private final static byte[] qual1 = Bytes.toBytes("qf_1"); + private final static byte[] qual2 = Bytes.toBytes("qf_2"); + public static Configuration conf; + + @Rule + public final TestName TEST_NAME = new TestName(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + conf.setBoolean("hbase.online.schema.update.enable", true); + conf.setInt("hbase.client.scanner.timeout.period", 600000); + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testHBASE11728() throws Exception { + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + HTableDescriptor desc = new HTableDescriptor(tableName); + colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE); + colDesc.setTimeToLive(15552000); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("a-b-0-0")); + put.add(fam, qual1, Bytes.toBytes("c1-value")); + table.put(put); + table.flushCommits(); + put = new Put(row1_bytes); + put.add(fam, qual1, Bytes.toBytes("c1-value")); + table.put(put); + table.flushCommits(); + put = new Put(row2_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value")); + table.put(put); + table.flushCommits(); + put = new Put(row3_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value-2")); + table.put(put); + table.flushCommits(); + put = new Put(row4_bytes); + put.add(fam, qual2, Bytes.toBytes("c2-value-3")); + table.put(put); + table.flushCommits(); + hBaseAdmin.flush(tableName.getNameAsString()); + String[] rows = new String[3]; + rows[0] = row1; + rows[1] = row2; + rows[2] = row3; + Scan scan = new Scan(); + scan.setStartRow(row1_bytes); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + ResultScanner scanner = table.getScanner(scan); + Result[] next = scanner.next(10); + assertEquals(3, next.length); + int i = 0; + for (Result res : next) { + CellScanner cellScanner = res.cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + } + i++; + } + scanner.close(); + // Add column + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(row1_bytes); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = table.getScanner(scan); + next = scanner.next(10); + assertEquals(2, next.length); + i = 1; + for (Result res : next) { + CellScanner cellScanner = res.cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + } + i++; + } + scanner.close(); + i = 1; + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(Bytes.toBytes("a-b-A-1-")); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = table.getScanner(scan); + next = scanner.next(10); + assertEquals(2, next.length); + for (Result res : next) { + CellScanner cellScanner = res.cellScanner(); + while (cellScanner.advance()) { + assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner + .current().getRowOffset(), cellScanner.current().getRowLength())); + } + i++; + } + scanner.close(); + scan = new Scan(); + scan.addColumn(fam, qual2); + scan.setStartRow(Bytes.toBytes("a-b-A-1-140239")); + scan.setStopRow(Bytes.toBytes("a-b-A-1:")); + scanner = table.getScanner(scan); + next = scanner.next(10); + assertEquals(1, next.length); + scanner.close(); + } finally { + table.close(); + } + } +}
