HBASE-16225 Refactor ScanQueryMatcher
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b35cf8f9 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b35cf8f9 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b35cf8f9 Branch: refs/heads/master Commit: b35cf8f93a54b73389431ea7ba2d08f4aae7e104 Parents: 7836124 Author: zhangduo <zhang...@apache.org> Authored: Sat Jul 30 23:50:24 2016 +0800 Committer: zhangduo <zhang...@apache.org> Committed: Tue Aug 2 14:01:28 2016 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/io/TimeRange.java | 7 +- .../hbase/coprocessor/BaseRegionObserver.java | 2 +- .../hbase/coprocessor/RegionObserver.java | 2 +- .../hadoop/hbase/regionserver/ColumnCount.java | 111 --- .../hbase/regionserver/ColumnTracker.java | 128 ---- .../hbase/regionserver/DeleteTracker.java | 106 --- .../regionserver/ExplicitColumnTracker.java | 254 ------- .../hadoop/hbase/regionserver/HStore.java | 30 +- .../regionserver/RegionCoprocessorHost.java | 1 + .../hbase/regionserver/ScanDeleteTracker.java | 174 ----- .../hadoop/hbase/regionserver/ScanInfo.java | 6 +- .../hbase/regionserver/ScanQueryMatcher.java | 699 ------------------- .../regionserver/ScanWildcardColumnTracker.java | 201 ------ .../apache/hadoop/hbase/regionserver/Store.java | 1 + .../hbase/regionserver/StoreFileReader.java | 50 +- .../hbase/regionserver/StoreFileScanner.java | 61 +- .../hadoop/hbase/regionserver/StoreScanner.java | 239 ++++--- .../regionserver/querymatcher/ColumnCount.java | 110 +++ .../querymatcher/ColumnTracker.java | 128 ++++ .../CompactionScanQueryMatcher.java | 119 ++++ .../querymatcher/DeleteTracker.java | 101 +++ .../DropDeletesCompactionScanQueryMatcher.java | 83 +++ .../querymatcher/ExplicitColumnTracker.java | 251 +++++++ .../querymatcher/LegacyScanQueryMatcher.java | 397 +++++++++++ .../MajorCompactionScanQueryMatcher.java | 80 +++ .../MinorCompactionScanQueryMatcher.java | 61 ++ .../NormalUserScanQueryMatcher.java | 106 +++ .../querymatcher/RawScanQueryMatcher.java | 77 ++ .../querymatcher/ScanDeleteTracker.java | 159 +++++ .../querymatcher/ScanQueryMatcher.java | 357 ++++++++++ .../querymatcher/ScanWildcardColumnTracker.java | 199 ++++++ .../StripeCompactionScanQueryMatcher.java | 118 ++++ .../querymatcher/UserScanQueryMatcher.java | 217 ++++++ .../visibility/VisibilityController.java | 2 +- .../visibility/VisibilityScanDeleteTracker.java | 2 +- .../regionserver/DataBlockEncodingTool.java | 2 +- .../EncodedSeekPerformanceTest.java | 4 +- .../regionserver/TestCompoundBloomFilter.java | 2 +- .../regionserver/TestExplicitColumnTracker.java | 195 ------ .../hbase/regionserver/TestQueryMatcher.java | 382 ---------- .../regionserver/TestScanDeleteTracker.java | 197 ------ .../TestScanWildcardColumnTracker.java | 130 ---- .../hbase/regionserver/TestStoreFile.java | 23 +- .../TestStoreFileScannerWithTagCompression.java | 2 +- .../hbase/regionserver/TestStoreScanner.java | 3 +- .../regionserver/compactions/TestCompactor.java | 4 +- .../compactions/TestStripeCompactionPolicy.java | 5 +- .../AbstractTestScanQueryMatcher.java | 77 ++ .../TestCompactionScanQueryMatcher.java | 99 +++ .../querymatcher/TestExplicitColumnTracker.java | 185 +++++ .../querymatcher/TestScanDeleteTracker.java | 185 +++++ .../TestScanWildcardColumnTracker.java | 126 ++++ .../querymatcher/TestUserScanQueryMatcher.java | 236 +++++++ 53 files changed, 3674 insertions(+), 2822 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java index fed20c4..2efcde1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java @@ -184,12 +184,11 @@ public class TimeRange { } /** - * Check if the specified timestamp is within this TimeRange. + * Check if the specified timestamp is within or after this TimeRange. * <p> - * Returns true if within interval [minStamp, maxStamp), false - * if not. + * Returns true if greater than minStamp, false if not. * @param timestamp timestamp to check - * @return true if within TimeRange, false if not + * @return true if within or after TimeRange, false if not */ public boolean withinOrAfterTimeRange(long timestamp) { assert timestamp >= 0; http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java index bff727a..748268e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.regionserver.DeleteTracker; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; @@ -57,6 +56,7 @@ import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 4729954..4c94644 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.regionserver.DeleteTracker; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -57,6 +56,7 @@ import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java deleted file mode 100644 index 71ea1bd..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * Simple wrapper for a byte buffer and a counter. Does not copy. - * <p> - * NOT thread-safe because it is not used in a multi-threaded context, yet. - */ -@InterfaceAudience.Private -public class ColumnCount { - private final byte [] bytes; - private final int offset; - private final int length; - private int count; - - /** - * Constructor - * @param column the qualifier to count the versions for - */ - public ColumnCount(byte [] column) { - this(column, 0); - } - - /** - * Constructor - * @param column the qualifier to count the versions for - * @param count initial count - */ - public ColumnCount(byte [] column, int count) { - this(column, 0, column.length, count); - } - - /** - * Constuctor - * @param column the qualifier to count the versions for - * @param offset in the passed buffer where to start the qualifier from - * @param length of the qualifier - * @param count initial count - */ - public ColumnCount(byte [] column, int offset, int length, int count) { - this.bytes = column; - this.offset = offset; - this.length = length; - this.count = count; - } - - /** - * @return the buffer - */ - public byte [] getBuffer(){ - return this.bytes; - } - - /** - * @return the offset - */ - public int getOffset(){ - return this.offset; - } - - /** - * @return the length - */ - public int getLength(){ - return this.length; - } - - /** - * Decrement the current version count - * @return current count - */ - public int decrement() { - return --count; - } - - /** - * Increment the current version count - * @return current count - */ - public int increment() { - return ++count; - } - - /** - * Set the current count to a new count - * @param count new count to set - */ - public void setCount(int count) { - this.count = count; - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java deleted file mode 100644 index d352561..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.io.IOException; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; - -/** - * Implementing classes of this interface will be used for the tracking - * and enforcement of columns and numbers of versions and timeToLive during - * the course of a Get or Scan operation. - * <p> - * Currently there are two different types of Store/Family-level queries. - * <ul><li>{@link ExplicitColumnTracker} is used when the query specifies - * one or more column qualifiers to return in the family.</li> - * <li>{@link ScanWildcardColumnTracker} is used when no columns are - * explicitly specified.</li> - * </ul> - * <p> - * This class is utilized by {@link ScanQueryMatcher} mainly through two methods: - * <ul><li>{@link #checkColumn} is called when a Put satisfies all other - * conditions of the query.</li> - * <li>{@link #getNextRowOrNextColumn} is called whenever ScanQueryMatcher - * believes that the current column should be skipped (by timestamp, filter etc.)</li> - * </ul> - * <p> - * These two methods returns a - * {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} - * to define what action should be taken. - * <p> - * This class is NOT thread-safe as queries are never multi-threaded - */ -@InterfaceAudience.Private -public interface ColumnTracker { - - /** - * Checks if the column is present in the list of requested columns by returning the match code - * instance. It does not check against the number of versions for the columns asked for. To do the - * version check, one has to call {@link #checkVersions(Cell, long, byte, boolean)} - * method based on the return type (INCLUDE) of this method. The values that can be returned by - * this method are {@link MatchCode#INCLUDE}, {@link MatchCode#SEEK_NEXT_COL} and - * {@link MatchCode#SEEK_NEXT_ROW}. - * @param cell - * @param type The type of the KeyValue - * @return The match code instance. - * @throws IOException in case there is an internal consistency problem caused by a data - * corruption. - */ - ScanQueryMatcher.MatchCode checkColumn(Cell cell, byte type) throws IOException; - - /** - * Keeps track of the number of versions for the columns asked for. It assumes that the user has - * already checked if the keyvalue needs to be included by calling the - * {@link #checkColumn(Cell, byte)} method. The enum values returned by this method - * are {@link MatchCode#SKIP}, {@link MatchCode#INCLUDE}, - * {@link MatchCode#INCLUDE_AND_SEEK_NEXT_COL} and {@link MatchCode#INCLUDE_AND_SEEK_NEXT_ROW}. - * Implementations which include all the columns could just return {@link MatchCode#INCLUDE} in - * the {@link #checkColumn(Cell, byte)} method and perform all the operations in this - * checkVersions method. - * @param cell - * @param ttl The timeToLive to enforce. - * @param type the type of the key value (Put/Delete) - * @param ignoreCount indicates if the KV needs to be excluded while counting (used during - * compactions. We only count KV's that are older than all the scanners' read points.) - * @return the scan query matcher match code instance - * @throws IOException in case there is an internal consistency problem caused by a data - * corruption. - */ - ScanQueryMatcher.MatchCode checkVersions(Cell cell, long ttl, byte type, boolean ignoreCount) - throws IOException; - /** - * Resets the Matcher - */ - void reset(); - - /** - * - * @return <code>true</code> when done. - */ - boolean done(); - - /** - * Used by matcher and scan/get to get a hint of the next column - * to seek to after checkColumn() returns SKIP. Returns the next interesting - * column we want, or NULL there is none (wildcard scanner). - * - * Implementations aren't required to return anything useful unless the most recent - * call was to checkColumn() and the return code was SKIP. This is pretty implementation - * detail-y, but optimizations are like that. - * - * @return null, or a ColumnCount that we should seek to - */ - ColumnCount getColumnHint(); - - /** - * Retrieve the MatchCode for the next row or column - * @param cell - */ - MatchCode getNextRowOrNextColumn(Cell cell); - - /** - * Give the tracker a chance to declare it's done based on only the timestamp - * to allow an early out. - * - * @param timestamp - * @return <code>true</code> to early out based on timestamp. - */ - boolean isDone(long timestamp); -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java deleted file mode 100644 index 8f466fc..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Cell; - -/** - * This interface is used for the tracking and enforcement of Deletes - * during the course of a Get or Scan operation. - * <p> - * This class is utilized through three methods: - * <ul><li>{@link #add} when encountering a Delete</li> - * <li>{@link #isDeleted} when checking if a Put KeyValue has been deleted</li> - * <li>{@link #update} when reaching the end of a StoreFile</li> - * </ul> - */ -@InterfaceAudience.Private -public interface DeleteTracker { - - /** - * Add the specified cell to the list of deletes to check against for - * this row operation. - * <p> - * This is called when a Delete is encountered in a StoreFile. - * @param cell - the delete cell - */ - void add(Cell cell); - - /** - * Check if the specified cell buffer has been deleted by a previously - * seen delete. - * @param cell - current cell to check if deleted by a previously seen delete - * @return deleteResult The result tells whether the KeyValue is deleted and why - */ - DeleteResult isDeleted(Cell cell); - - /** - * @return true if there are no current delete, false otherwise - */ - boolean isEmpty(); - - /** - * Called at the end of every StoreFile. - * <p> - * Many optimized implementations of Trackers will require an update at - * when the end of each StoreFile is reached. - */ - void update(); - - /** - * Called between rows. - * <p> - * This clears everything as if a new DeleteTracker was instantiated. - */ - void reset(); - - - /** - * Return codes for comparison of two Deletes. - * <p> - * The codes tell the merging function what to do. - * <p> - * INCLUDE means add the specified Delete to the merged list. - * NEXT means move to the next element in the specified list(s). - */ - enum DeleteCompare { - INCLUDE_OLD_NEXT_OLD, - INCLUDE_OLD_NEXT_BOTH, - INCLUDE_NEW_NEXT_NEW, - INCLUDE_NEW_NEXT_BOTH, - NEXT_OLD, - NEXT_NEW - } - - /** - * Returns codes for delete result. - * The codes tell the ScanQueryMatcher whether the kv is deleted and why. - * Based on the delete result, the ScanQueryMatcher will decide the next - * operation - */ - enum DeleteResult { - FAMILY_DELETED, // The KeyValue is deleted by a delete family. - FAMILY_VERSION_DELETED, // The KeyValue is deleted by a delete family version. - COLUMN_DELETED, // The KeyValue is deleted by a delete column. - VERSION_DELETED, // The KeyValue is deleted by a version delete. - NOT_DELETED - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java deleted file mode 100644 index 86c8b48..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.io.IOException; -import java.util.NavigableSet; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; - -/** - * This class is used for the tracking and enforcement of columns and numbers - * of versions during the course of a Get or Scan operation, when explicit - * column qualifiers have been asked for in the query. - * - * With a little magic (see {@link ScanQueryMatcher}), we can use this matcher - * for both scans and gets. The main difference is 'next' and 'done' collapse - * for the scan case (since we see all columns in order), and we only reset - * between rows. - * - * <p> - * This class is utilized by {@link ScanQueryMatcher} mainly through two methods: - * <ul><li>{@link #checkColumn} is called when a Put satisfies all other - * conditions of the query.</li> - * <li>{@link #getNextRowOrNextColumn} is called whenever ScanQueryMatcher - * believes that the current column should be skipped (by timestamp, filter etc.)</li> - * </ul> - * <p> - * These two methods returns a - * {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} - * to define what action should be taken. - * <p> - * This class is NOT thread-safe as queries are never multi-threaded - */ -@InterfaceAudience.Private -public class ExplicitColumnTracker implements ColumnTracker { - - private final int maxVersions; - private final int minVersions; - - /** - * Contains the list of columns that the ExplicitColumnTracker is tracking. - * Each ColumnCount instance also tracks how many versions of the requested - * column have been returned. - */ - private final ColumnCount[] columns; - private int index; - private ColumnCount column; - /** Keeps track of the latest timestamp included for current column. - * Used to eliminate duplicates. */ - private long latestTSOfCurrentColumn; - private long oldestStamp; - - /** - * Default constructor. - * @param columns columns specified user in query - * @param minVersions minimum number of versions to keep - * @param maxVersions maximum versions to return per column - * @param oldestUnexpiredTS the oldest timestamp we are interested in, - * based on TTL - */ - public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions, - int maxVersions, long oldestUnexpiredTS) { - this.maxVersions = maxVersions; - this.minVersions = minVersions; - this.oldestStamp = oldestUnexpiredTS; - this.columns = new ColumnCount[columns.size()]; - int i=0; - for(byte [] column : columns) { - this.columns[i++] = new ColumnCount(column); - } - reset(); - } - - /** - * Done when there are no more columns to match against. - */ - public boolean done() { - return this.index >= columns.length; - } - - public ColumnCount getColumnHint() { - return this.column; - } - - /** - * {@inheritDoc} - */ - @Override - public ScanQueryMatcher.MatchCode checkColumn(Cell cell, byte type) { - // delete markers should never be passed to an - // *Explicit*ColumnTracker - assert !CellUtil.isDelete(type); - do { - // No more columns left, we are done with this query - if(done()) { - return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row - } - - // No more columns to match against, done with storefile - if(this.column == null) { - return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row - } - - // Compare specific column to current column - int ret = CellComparator.compareQualifiers(cell, column.getBuffer(), column.getOffset(), - column.getLength()); - - // Column Matches. Return include code. The caller would call checkVersions - // to limit the number of versions. - if (ret == 0) { - return ScanQueryMatcher.MatchCode.INCLUDE; - } - - resetTS(); - - if (ret < 0) { - // The current KV is smaller than the column the ExplicitColumnTracker - // is interested in, so seek to that column of interest. - return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; - } - - // The current KV is bigger than the column the ExplicitColumnTracker - // is interested in. That means there is no more data for the column - // of interest. Advance the ExplicitColumnTracker state to next - // column of interest, and check again. - if (ret > 0) { - ++this.index; - if (done()) { - // No more to match, do not include, done with this row. - return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row - } - // This is the recursive case. - this.column = this.columns[this.index]; - } - } while(true); - } - - @Override - public ScanQueryMatcher.MatchCode checkVersions(Cell cell, - long timestamp, byte type, boolean ignoreCount) throws IOException { - assert !CellUtil.isDelete(type); - if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; - // Check if it is a duplicate timestamp - if (sameAsPreviousTS(timestamp)) { - // If duplicate, skip this Key - return ScanQueryMatcher.MatchCode.SKIP; - } - int count = this.column.increment(); - if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) { - // Done with versions for this column - ++this.index; - resetTS(); - if (done()) { - // We have served all the requested columns. - this.column = null; - return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; - } - // We are done with current column; advance to next column - // of interest. - this.column = this.columns[this.index]; - return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL; - } - setTS(timestamp); - return ScanQueryMatcher.MatchCode.INCLUDE; - } - - // Called between every row. - public void reset() { - this.index = 0; - this.column = this.columns[this.index]; - for(ColumnCount col : this.columns) { - col.setCount(0); - } - resetTS(); - } - - private void resetTS() { - latestTSOfCurrentColumn = HConstants.LATEST_TIMESTAMP; - } - - private void setTS(long timestamp) { - latestTSOfCurrentColumn = timestamp; - } - - private boolean sameAsPreviousTS(long timestamp) { - return timestamp == latestTSOfCurrentColumn; - } - - private boolean isExpired(long timestamp) { - return timestamp < oldestStamp; - } - - /** - * This method is used to inform the column tracker that we are done with - * this column. We may get this information from external filters or - * timestamp range and we then need to indicate this information to - * tracker. It is required only in case of ExplicitColumnTracker. - * @param cell - */ - public void doneWithColumn(Cell cell) { - while (this.column != null) { - int compare = CellComparator.compareQualifiers(cell, column.getBuffer(), column.getOffset(), - column.getLength()); - resetTS(); - if (compare >= 0) { - ++this.index; - if (done()) { - // Will not hit any more columns in this storefile - this.column = null; - } else { - this.column = this.columns[this.index]; - } - if (compare > 0) continue; - } - return; - } - } - - @Override - public MatchCode getNextRowOrNextColumn(Cell cell) { - doneWithColumn(cell); - - if (getColumnHint() == null) { - return MatchCode.SEEK_NEXT_ROW; - } else { - return MatchCode.SEEK_NEXT_COL; - } - } - - public boolean isDone(long timestamp) { - return minVersions <= 0 && isExpired(timestamp); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index b32b757..c4bd849 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor; import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; +import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.EncryptionUtil; @@ -1749,35 +1750,6 @@ public class HStore implements Store { return wantedVersions > maxVersions ? maxVersions: wantedVersions; } - /** - * @param cell - * @param oldestTimestamp - * @return true if the cell is expired - */ - static boolean isCellTTLExpired(final Cell cell, final long oldestTimestamp, final long now) { - // Look for a TTL tag first. Use it instead of the family setting if - // found. If a cell has multiple TTLs, resolve the conflict by using the - // first tag encountered. - Iterator<Tag> i = CellUtil.tagsIterator(cell); - while (i.hasNext()) { - Tag t = i.next(); - if (TagType.TTL_TAG_TYPE == t.getType()) { - // Unlike in schema cell TTLs are stored in milliseconds, no need - // to convert - long ts = cell.getTimestamp(); - assert t.getValueLength() == Bytes.SIZEOF_LONG; - long ttl = TagUtil.getValueAsLong(t); - if (ts + ttl < now) { - return true; - } - // Per cell TTLs cannot extend lifetime beyond family settings, so - // fall through to check that - break; - } - } - return false; - } - @Override public boolean canSplit() { this.lock.readLock().lock(); http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index acfaa96..567664e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -77,6 +77,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.wal.WALKey; http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java deleted file mode 100644 index e2db008..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java +++ /dev/null @@ -1,174 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver; - -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * This class is responsible for the tracking and enforcement of Deletes - * during the course of a Scan operation. - * - * It only has to enforce Delete and DeleteColumn, since the - * DeleteFamily is handled at a higher level. - * - * <p> - * This class is utilized through three methods: - * <ul><li>{@link #add} when encountering a Delete or DeleteColumn</li> - * <li>{@link #isDeleted} when checking if a Put KeyValue has been deleted</li> - * <li>{@link #update} when reaching the end of a StoreFile or row for scans</li> - * </ul> - * <p> - * This class is NOT thread-safe as queries are never multi-threaded - */ -@InterfaceAudience.Private -public class ScanDeleteTracker implements DeleteTracker { - - protected boolean hasFamilyStamp = false; - protected long familyStamp = 0L; - protected SortedSet<Long> familyVersionStamps = new TreeSet<Long>(); - protected byte [] deleteBuffer = null; - protected int deleteOffset = 0; - protected int deleteLength = 0; - protected byte deleteType = 0; - protected long deleteTimestamp = 0L; - - /** - * Constructor for ScanDeleteTracker - */ - public ScanDeleteTracker() { - super(); - } - - /** - * Add the specified KeyValue to the list of deletes to check against for - * this row operation. - * <p> - * This is called when a Delete is encountered. - * @param cell - the delete cell - */ - @Override - public void add(Cell cell) { - long timestamp = cell.getTimestamp(); - byte type = cell.getTypeByte(); - if (!hasFamilyStamp || timestamp > familyStamp) { - if (type == KeyValue.Type.DeleteFamily.getCode()) { - hasFamilyStamp = true; - familyStamp = timestamp; - return; - } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) { - familyVersionStamps.add(timestamp); - return; - } - - if (deleteBuffer != null && type < deleteType) { - // same column, so ignore less specific delete - if (CellUtil.matchingQualifier(cell, deleteBuffer, deleteOffset, deleteLength)) { - return; - } - } - // new column, or more general delete type - deleteBuffer = cell.getQualifierArray(); - deleteOffset = cell.getQualifierOffset(); - deleteLength = cell.getQualifierLength(); - deleteType = type; - deleteTimestamp = timestamp; - } - // missing else is never called. - } - - /** - * Check if the specified KeyValue buffer has been deleted by a previously - * seen delete. - * - * @param cell - current cell to check if deleted by a previously seen delete - * @return deleteResult - */ - @Override - public DeleteResult isDeleted(Cell cell) { - long timestamp = cell.getTimestamp(); - if (hasFamilyStamp && timestamp <= familyStamp) { - return DeleteResult.FAMILY_DELETED; - } - - if (familyVersionStamps.contains(Long.valueOf(timestamp))) { - return DeleteResult.FAMILY_VERSION_DELETED; - } - - if (deleteBuffer != null) { - int ret = -(CellComparator.compareQualifiers(cell, deleteBuffer, deleteOffset, deleteLength)); - if (ret == 0) { - if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { - return DeleteResult.COLUMN_DELETED; - } - // Delete (aka DeleteVersion) - // If the timestamp is the same, keep this one - if (timestamp == deleteTimestamp) { - return DeleteResult.VERSION_DELETED; - } - // use assert or not? - assert timestamp < deleteTimestamp; - - // different timestamp, let's clear the buffer. - deleteBuffer = null; - } else if(ret < 0){ - // Next column case. - deleteBuffer = null; - } else { - throw new IllegalStateException("isDelete failed: deleteBuffer=" - + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) - + ", qualifier=" - + Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength()) - + ", timestamp=" + timestamp + ", comparison result: " + ret); - } - } - - return DeleteResult.NOT_DELETED; - } - - @Override - public boolean isEmpty() { - return deleteBuffer == null && !hasFamilyStamp && - familyVersionStamps.isEmpty(); - } - - @Override - // called between every row. - public void reset() { - hasFamilyStamp = false; - familyStamp = 0L; - familyVersionStamps.clear(); - deleteBuffer = null; - } - - @Override - // should not be called at all even (!) - public void update() { - this.reset(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java index 42f7369..349e166 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java @@ -122,11 +122,11 @@ public class ScanInfo { return this.parallelSeekEnabled; } - byte[] getFamily() { + public byte[] getFamily() { return family; } - int getMinVersions() { + public int getMinVersions() { return minVersions; } @@ -138,7 +138,7 @@ public class ScanInfo { return ttl; } - KeepDeletedCells getKeepDeletedCells() { + public KeepDeletedCells getKeepDeletedCells() { return keepDeletedCells; } http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java deleted file mode 100644 index d2d0ccb..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ /dev/null @@ -1,699 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver; - -import java.io.IOException; -import java.util.Arrays; -import java.util.NavigableSet; - -import org.apache.hadoop.hbase.KeyValue.Type; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.Filter.ReturnCode; -import org.apache.hadoop.hbase.io.TimeRange; -import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -import com.google.common.base.Preconditions; - -/** - * A query matcher that is specifically designed for the scan case. - */ -@InterfaceAudience.Private -public class ScanQueryMatcher { - // Optimization so we can skip lots of compares when we decide to skip - // to the next row. - private boolean stickyNextRow; - private final byte[] stopRow; - - private final TimeRange tr; - - private final Filter filter; - - /** Keeps track of deletes */ - private final DeleteTracker deletes; - - /* - * The following three booleans define how we deal with deletes. - * There are three different aspects: - * 1. Whether to keep delete markers. This is used in compactions. - * Minor compactions always keep delete markers. - * 2. Whether to keep deleted rows. This is also used in compactions, - * if the store is set to keep deleted rows. This implies keeping - * the delete markers as well. - * In this case deleted rows are subject to the normal max version - * and TTL/min version rules just like "normal" rows. - * 3. Whether a scan can do time travel queries even before deleted - * marker to reach deleted rows. - */ - /** whether to retain delete markers */ - private boolean retainDeletesInOutput; - - /** whether to return deleted rows */ - private final KeepDeletedCells keepDeletedCells; - /** whether time range queries can see rows "behind" a delete */ - private final boolean seePastDeleteMarkers; - - - /** Keeps track of columns and versions */ - private final ColumnTracker columns; - - /** Key to seek to in memstore and StoreFiles */ - private final Cell startKey; - - /** Row comparator for the region this query is for */ - private final CellComparator rowComparator; - - /* row is not private for tests */ - /** Row the query is on */ - Cell curCell; - - /** - * Oldest put in any of the involved store files - * Used to decide whether it is ok to delete - * family delete marker of this store keeps - * deleted KVs. - */ - private final long earliestPutTs; - private final long ttl; - - /** The oldest timestamp we are interested in, based on TTL */ - private final long oldestUnexpiredTS; - private final long now; - - /** readPoint over which the KVs are unconditionally included */ - protected long maxReadPointToTrackVersions; - - private byte[] dropDeletesFromRow = null, dropDeletesToRow = null; - - /** - * This variable shows whether there is an null column in the query. There - * always exists a null column in the wildcard column query. - * There maybe exists a null column in the explicit column query based on the - * first column. - * */ - private boolean hasNullColumn = true; - - private RegionCoprocessorHost regionCoprocessorHost= null; - - // By default, when hbase.hstore.time.to.purge.deletes is 0ms, a delete - // marker is always removed during a major compaction. If set to non-zero - // value then major compaction will try to keep a delete marker around for - // the given number of milliseconds. We want to keep the delete markers - // around a bit longer because old puts might appear out-of-order. For - // example, during log replication between two clusters. - // - // If the delete marker has lived longer than its column-family's TTL then - // the delete marker will be removed even if time.to.purge.deletes has not - // passed. This is because all the Puts that this delete marker can influence - // would have also expired. (Removing of delete markers on col family TTL will - // not happen if min-versions is set to non-zero) - // - // But, if time.to.purge.deletes has not expired then a delete - // marker will not be removed just because there are no Puts that it is - // currently influencing. This is because Puts, that this delete can - // influence. may appear out of order. - private final long timeToPurgeDeletes; - - private final boolean isUserScan; - - private final boolean isReversed; - - /** - * True if we are doing a 'Get' Scan. Every Get is actually a one-row Scan. - */ - private final boolean get; - - /** - * Construct a QueryMatcher for a scan - * @param scanInfo The store's immutable scan info - * @param scanType Type of the scan - * @param earliestPutTs Earliest put seen in any of the store files. - * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL - */ - public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns, - ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, - long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException { - TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily()); - if (timeRange == null) { - this.tr = scan.getTimeRange(); - } else { - this.tr = timeRange; - } - this.get = scan.isGetScan(); - this.rowComparator = scanInfo.getComparator(); - this.regionCoprocessorHost = regionCoprocessorHost; - this.deletes = instantiateDeleteTracker(); - this.stopRow = scan.getStopRow(); - this.startKey = CellUtil.createFirstDeleteFamilyCellOnRow(scan.getStartRow(), - scanInfo.getFamily()); - this.filter = scan.getFilter(); - this.earliestPutTs = earliestPutTs; - this.oldestUnexpiredTS = oldestUnexpiredTS; - this.now = now; - - this.maxReadPointToTrackVersions = readPointToUse; - this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); - this.ttl = oldestUnexpiredTS; - - /* how to deal with deletes */ - this.isUserScan = scanType == ScanType.USER_SCAN; - // keep deleted cells: if compaction or raw scan - this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE : - isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells(); - // retain deletes: if minor compaction or raw scanisDone - this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw(); - // seePastDeleteMarker: user initiated scans - this.seePastDeleteMarkers = - scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan; - - int maxVersions = - scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), - scanInfo.getMaxVersions()); - - // Single branch to deal with two types of reads (columns vs all in family) - if (columns == null || columns.size() == 0) { - // there is always a null column in the wildcard column query. - hasNullColumn = true; - - // use a specialized scan for wildcard column tracker. - this.columns = new ScanWildcardColumnTracker( - scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS); - } else { - // whether there is null column in the explicit column query - hasNullColumn = (columns.first().length == 0); - - // We can share the ExplicitColumnTracker, diff is we reset - // between rows, not between storefiles. - this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions, - oldestUnexpiredTS); - } - this.isReversed = scan.isReversed(); - } - - private DeleteTracker instantiateDeleteTracker() throws IOException { - DeleteTracker tracker = new ScanDeleteTracker(); - if (regionCoprocessorHost != null) { - tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker); - } - return tracker; - } - - /** - * Construct a QueryMatcher for a scan that drop deletes from a limited range of rows. - * @param scan - * @param scanInfo The store's immutable scan info - * @param columns - * @param earliestPutTs Earliest put seen in any of the store files. - * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL - * @param now the current server time - * @param dropDeletesFromRow The inclusive left bound of the range; can be EMPTY_START_ROW. - * @param dropDeletesToRow The exclusive right bound of the range; can be EMPTY_END_ROW. - * @param regionCoprocessorHost - * @throws IOException - */ - public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns, - long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now, - byte[] dropDeletesFromRow, byte[] dropDeletesToRow, - RegionCoprocessorHost regionCoprocessorHost) throws IOException { - this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs, - oldestUnexpiredTS, now, regionCoprocessorHost); - Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null)); - this.dropDeletesFromRow = dropDeletesFromRow; - this.dropDeletesToRow = dropDeletesToRow; - } - - /* - * Constructor for tests - */ - ScanQueryMatcher(Scan scan, ScanInfo scanInfo, - NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException { - this(scan, scanInfo, columns, ScanType.USER_SCAN, - Long.MAX_VALUE, /* max Readpoint to track versions */ - HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null); - } - - /** - * - * @return whether there is an null column in the query - */ - public boolean hasNullColumnInQuery() { - return hasNullColumn; - } - - /** - * Determines if the caller should do one of several things: - * - seek/skip to the next row (MatchCode.SEEK_NEXT_ROW) - * - seek/skip to the next column (MatchCode.SEEK_NEXT_COL) - * - include the current KeyValue (MatchCode.INCLUDE) - * - ignore the current KeyValue (MatchCode.SKIP) - * - got to the next row (MatchCode.DONE) - * - * @param cell KeyValue to check - * @return The match code instance. - * @throws IOException in case there is an internal consistency problem - * caused by a data corruption. - */ - public MatchCode match(Cell cell) throws IOException { - if (filter != null && filter.filterAllRemaining()) { - return MatchCode.DONE_SCAN; - } - if (curCell != null) { - int ret = this.rowComparator.compareRows(curCell, cell); - if (!this.isReversed) { - if (ret <= -1) { - return MatchCode.DONE; - } else if (ret >= 1) { - // could optimize this, if necessary? - // Could also be called SEEK_TO_CURRENT_ROW, but this - // should be rare/never happens. - return MatchCode.SEEK_NEXT_ROW; - } - } else { - if (ret <= -1) { - return MatchCode.SEEK_NEXT_ROW; - } else if (ret >= 1) { - return MatchCode.DONE; - } - } - } else { - // Since the curCell is null it means we are already sure that we have moved over to the next row - return MatchCode.DONE; - } - - // optimize case. - if (this.stickyNextRow) { - return MatchCode.SEEK_NEXT_ROW; - } - - if (this.columns.done()) { - stickyNextRow = true; - return MatchCode.SEEK_NEXT_ROW; - } - - long timestamp = cell.getTimestamp(); - // check for early out based on timestamp alone - if (columns.isDone(timestamp)) { - return columns.getNextRowOrNextColumn(cell); - } - // check if the cell is expired by cell TTL - if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) { - return MatchCode.SKIP; - } - - /* - * The delete logic is pretty complicated now. - * This is corroborated by the following: - * 1. The store might be instructed to keep deleted rows around. - * 2. A scan can optionally see past a delete marker now. - * 3. If deleted rows are kept, we have to find out when we can - * remove the delete markers. - * 4. Family delete markers are always first (regardless of their TS) - * 5. Delete markers should not be counted as version - * 6. Delete markers affect puts of the *same* TS - * 7. Delete marker need to be version counted together with puts - * they affect - */ - byte typeByte = cell.getTypeByte(); - long mvccVersion = cell.getSequenceId(); - if (CellUtil.isDelete(cell)) { - if (keepDeletedCells == KeepDeletedCells.FALSE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) { - // first ignore delete markers if the scanner can do so, and the - // range does not include the marker - // - // during flushes and compactions also ignore delete markers newer - // than the readpoint of any open scanner, this prevents deleted - // rows that could still be seen by a scanner from being collected - boolean includeDeleteMarker = seePastDeleteMarkers ? - tr.withinTimeRange(timestamp) : - tr.withinOrAfterTimeRange(timestamp); - if (includeDeleteMarker - && mvccVersion <= maxReadPointToTrackVersions) { - this.deletes.add(cell); - } - // Can't early out now, because DelFam come before any other keys - } - - if ((!isUserScan) - && timeToPurgeDeletes > 0 - && (EnvironmentEdgeManager.currentTime() - timestamp) - <= timeToPurgeDeletes) { - return MatchCode.INCLUDE; - } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) { - // always include or it is not time yet to check whether it is OK - // to purge deltes or not - if (!isUserScan) { - // if this is not a user scan (compaction), we can filter this deletemarker right here - // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking - return MatchCode.INCLUDE; - } - } else if (keepDeletedCells == KeepDeletedCells.TRUE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) { - if (timestamp < earliestPutTs) { - // keeping delete rows, but there are no puts older than - // this delete in the store files. - return columns.getNextRowOrNextColumn(cell); - } - // else: fall through and do version counting on the - // delete markers - } else { - return MatchCode.SKIP; - } - // note the following next else if... - // delete marker are not subject to other delete markers - } else if (!this.deletes.isEmpty()) { - DeleteResult deleteResult = deletes.isDeleted(cell); - switch (deleteResult) { - case FAMILY_DELETED: - case COLUMN_DELETED: - return columns.getNextRowOrNextColumn(cell); - case VERSION_DELETED: - case FAMILY_VERSION_DELETED: - return MatchCode.SKIP; - case NOT_DELETED: - break; - default: - throw new RuntimeException("UNEXPECTED"); - } - } - - // NOTE: Cryptic stuff! - // if the timestamp is HConstants.OLDEST_TIMESTAMP, then this is a fake cell made to prime a - // Scanner; See KeyValueUTil#createLastOnRow. This Cell should never end up returning out of - // here a matchcode of INCLUDE else we will return to the client a fake Cell. If we call - // TimeRange, it will return 0 because it doesn't deal in OLDEST_TIMESTAMP and we will fall - // into the later code where we could return a matchcode of INCLUDE. See HBASE-16074 "ITBLL - // fails, reports lost big or tiny families" for a horror story. Check here for - // OLDEST_TIMESTAMP. TimeRange#compare is about more generic timestamps, between 0L and - // Long.MAX_LONG. It doesn't do OLDEST_TIMESTAMP weird handling. - int timestampComparison = timestamp == HConstants.OLDEST_TIMESTAMP? -1: tr.compare(timestamp); - if (timestampComparison >= 1) { - return MatchCode.SKIP; - } else if (timestampComparison <= -1) { - return columns.getNextRowOrNextColumn(cell); - } - - // STEP 1: Check if the column is part of the requested columns - MatchCode colChecker = columns.checkColumn(cell, typeByte); - if (colChecker == MatchCode.INCLUDE) { - ReturnCode filterResponse = ReturnCode.SKIP; - // STEP 2: Yes, the column is part of the requested columns. Check if filter is present - if (filter != null) { - // STEP 3: Filter the key value and return if it filters out - filterResponse = filter.filterKeyValue(cell); - switch (filterResponse) { - case SKIP: - return MatchCode.SKIP; - case NEXT_COL: - return columns.getNextRowOrNextColumn(cell); - case NEXT_ROW: - stickyNextRow = true; - return MatchCode.SEEK_NEXT_ROW; - case SEEK_NEXT_USING_HINT: - return MatchCode.SEEK_NEXT_USING_HINT; - default: - //It means it is either include or include and seek next - break; - } - } - /* - * STEP 4: Reaching this step means the column is part of the requested columns and either - * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response. - * Now check the number of versions needed. This method call returns SKIP, INCLUDE, - * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL. - * - * FilterResponse ColumnChecker Desired behavior - * INCLUDE SKIP row has already been included, SKIP. - * INCLUDE INCLUDE INCLUDE - * INCLUDE INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW - * INCLUDE_AND_SEEK_NEXT_COL SKIP row has already been included, SKIP. - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW - * - * In all the above scenarios, we return the column checker return value except for - * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE) - */ - colChecker = columns.checkVersions(cell, timestamp, typeByte, - mvccVersion > maxReadPointToTrackVersions); - //Optimize with stickyNextRow - boolean seekNextRowFromEssential = filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW && - filter.isFamilyEssential(cell.getFamilyArray()); - if (colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW || seekNextRowFromEssential) { - stickyNextRow = true; - } - if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) { - if (colChecker != MatchCode.SKIP) { - return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; - } - return MatchCode.SEEK_NEXT_ROW; - } - return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && - colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL - : colChecker; - } - stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true - : stickyNextRow; - return colChecker; - } - - /** Handle partial-drop-deletes. As we match keys in order, when we have a range from which - * we can drop deletes, we can set retainDeletesInOutput to false for the duration of this - * range only, and maintain consistency. */ - private void checkPartialDropDeleteRange(Cell curCell) { - // If partial-drop-deletes are used, initially, dropDeletesFromRow and dropDeletesToRow - // are both set, and the matcher is set to retain deletes. We assume ordered keys. When - // dropDeletesFromRow is leq current kv, we start dropping deletes and reset - // dropDeletesFromRow; thus the 2nd "if" starts to apply. - if ((dropDeletesFromRow != null) - && (Arrays.equals(dropDeletesFromRow, HConstants.EMPTY_START_ROW) || - (CellComparator.COMPARATOR.compareRows(curCell, dropDeletesFromRow, 0, - dropDeletesFromRow.length) >= 0))) { - retainDeletesInOutput = false; - dropDeletesFromRow = null; - } - // If dropDeletesFromRow is null and dropDeletesToRow is set, we are inside the partial- - // drop-deletes range. When dropDeletesToRow is leq current kv, we stop dropping deletes, - // and reset dropDeletesToRow so that we don't do any more compares. - if ((dropDeletesFromRow == null) - && (dropDeletesToRow != null) - && !Arrays.equals(dropDeletesToRow, HConstants.EMPTY_END_ROW) - && (CellComparator.COMPARATOR - .compareRows(curCell, dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) { - retainDeletesInOutput = true; - dropDeletesToRow = null; - } - } - - /** - * @return Returns false if we know there are no more rows to be scanned (We've reached the - * <code>stopRow</code> or we are scanning on row only because this Scan is for a Get, etc. - */ - public boolean moreRowsMayExistAfter(Cell kv) { - // If a 'get' Scan -- we are doing a Get (every Get is a single-row Scan in implementation) -- - // then we are looking at one row only, the one specified in the Get coordinate..so we know - // for sure that there are no more rows on this Scan - if (this.get) { - return false; - } - // If no stopRow, return that there may be more rows. The tests that follow depend on a - // non-empty, non-default stopRow so this little test below short-circuits out doing the - // following compares. - if (this.stopRow == null || this.stopRow == HConstants.EMPTY_BYTE_ARRAY) { - return true; - } - return this.isReversed? - rowComparator.compareRows(kv, stopRow, 0, stopRow.length) > 0: - Bytes.equals(stopRow, HConstants.EMPTY_END_ROW) || - rowComparator.compareRows(kv, stopRow, 0, stopRow.length) < 0; - } - - /** - * Set the row when there is change in row - * @param curCell - */ - public void setToNewRow(Cell curCell) { - checkPartialDropDeleteRange(curCell); - this.curCell = curCell; - reset(); - } - - public void reset() { - this.deletes.reset(); - this.columns.reset(); - - stickyNextRow = false; - } - - /** - * - * @return the start key - */ - public Cell getStartKey() { - return this.startKey; - } - - /** - * - * @return the Filter - */ - Filter getFilter() { - return this.filter; - } - - public Cell getNextKeyHint(Cell kv) throws IOException { - if (filter == null) { - return null; - } else { - return filter.getNextCellHint(kv); - } - } - - public Cell getKeyForNextColumn(Cell kv) { - ColumnCount nextColumn = columns.getColumnHint(); - if (nextColumn == null) { - return CellUtil.createLastOnRowCol(kv); - } else { - return CellUtil.createFirstOnRowCol(kv, nextColumn.getBuffer(), nextColumn.getOffset(), - nextColumn.getLength()); - } - } - - public Cell getKeyForNextRow(Cell c) { - return CellUtil.createLastOnRow(c); - } - - /** - * @param nextIndexed the key of the next entry in the block index (if any) - * @param kv The Cell we're using to calculate the seek key - * @return result of the compare between the indexed key and the key portion of the passed cell - */ - public int compareKeyForNextRow(Cell nextIndexed, Cell kv) { - return rowComparator.compareKeyBasedOnColHint(nextIndexed, kv, 0, 0, null, 0, 0, - HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode()); - } - - /** - * @param nextIndexed the key of the next entry in the block index (if any) - * @param currentCell The Cell we're using to calculate the seek key - * @return result of the compare between the indexed key and the key portion of the passed cell - */ - public int compareKeyForNextColumn(Cell nextIndexed, Cell currentCell) { - ColumnCount nextColumn = columns.getColumnHint(); - if (nextColumn == null) { - return rowComparator.compareKeyBasedOnColHint(nextIndexed, currentCell, 0, 0, null, 0, 0, - HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode()); - } else { - return rowComparator.compareKeyBasedOnColHint(nextIndexed, currentCell, - currentCell.getFamilyOffset(), currentCell.getFamilyLength(), nextColumn.getBuffer(), - nextColumn.getOffset(), nextColumn.getLength(), HConstants.LATEST_TIMESTAMP, - Type.Maximum.getCode()); - } - } - - boolean isUserScan() { - return this.isUserScan; - } - - //Used only for testing purposes - static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, - int length, long ttl, byte type, boolean ignoreCount) throws IOException { - KeyValue kv = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY, 0, 0, - HConstants.EMPTY_BYTE_ARRAY, 0, 0, bytes, offset, length); - MatchCode matchCode = columnTracker.checkColumn(kv, type); - if (matchCode == MatchCode.INCLUDE) { - return columnTracker.checkVersions(kv, ttl, type, ignoreCount); - } - return matchCode; - } - - /** - * {@link #match} return codes. These instruct the scanner moving through - * memstores and StoreFiles what to do with the current KeyValue. - * <p> - * Additionally, this contains "early-out" language to tell the scanner to - * move on to the next File (memstore or Storefile), or to return immediately. - */ - public static enum MatchCode { - /** - * Include KeyValue in the returned result - */ - INCLUDE, - - /** - * Do not include KeyValue in the returned result - */ - SKIP, - - /** - * Do not include, jump to next StoreFile or memstore (in time order) - */ - NEXT, - - /** - * Do not include, return current result - */ - DONE, - - /** - * These codes are used by the ScanQueryMatcher - */ - - /** - * Done with the row, seek there. - */ - SEEK_NEXT_ROW, - /** - * Done with column, seek to next. - */ - SEEK_NEXT_COL, - - /** - * Done with scan, thanks to the row filter. - */ - DONE_SCAN, - - /* - * Seek to next key which is given as hint. - */ - SEEK_NEXT_USING_HINT, - - /** - * Include KeyValue and done with column, seek to next. - */ - INCLUDE_AND_SEEK_NEXT_COL, - - /** - * Include KeyValue and done with row, seek to next. - */ - INCLUDE_AND_SEEK_NEXT_ROW, - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java deleted file mode 100644 index b5ef319..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java +++ /dev/null @@ -1,201 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver; - -import java.io.IOException; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Keeps track of the columns for a scan if they are not explicitly specified - */ -@InterfaceAudience.Private -public class ScanWildcardColumnTracker implements ColumnTracker { - private Cell columnCell = null; - private int currentCount = 0; - private int maxVersions; - private int minVersions; - /* Keeps track of the latest timestamp and type included for current column. - * Used to eliminate duplicates. */ - private long latestTSOfCurrentColumn; - private byte latestTypeOfCurrentColumn; - - private long oldestStamp; - - /** - * Return maxVersions of every row. - * @param minVersion Minimum number of versions to keep - * @param maxVersion Maximum number of versions to return - * @param oldestUnexpiredTS oldest timestamp that has not expired according - * to the TTL. - */ - public ScanWildcardColumnTracker(int minVersion, int maxVersion, - long oldestUnexpiredTS) { - this.maxVersions = maxVersion; - this.minVersions = minVersion; - this.oldestStamp = oldestUnexpiredTS; - } - - /** - * {@inheritDoc} - * This receives puts *and* deletes. - */ - @Override - public MatchCode checkColumn(Cell cell, byte type) throws IOException { - return MatchCode.INCLUDE; - } - - /** - * {@inheritDoc} - * This receives puts *and* deletes. Deletes do not count as a version, but rather - * take the version of the previous put (so eventually all but the last can be reclaimed). - */ - @Override - public ScanQueryMatcher.MatchCode checkVersions(Cell cell, - long timestamp, byte type, boolean ignoreCount) throws IOException { - - if (columnCell == null) { - // first iteration. - resetCell(cell); - if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; - // do not count a delete marker as another version - return checkVersion(type, timestamp); - } - int cmp = CellComparator.compareQualifiers(cell, this.columnCell); - if (cmp == 0) { - if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; - - //If column matches, check if it is a duplicate timestamp - if (sameAsPreviousTSAndType(timestamp, type)) { - return ScanQueryMatcher.MatchCode.SKIP; - } - return checkVersion(type, timestamp); - } - - resetTSAndType(); - - // new col > old col - if (cmp > 0) { - // switched columns, lets do something.x - resetCell(cell); - if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; - return checkVersion(type, timestamp); - } - - // new col < oldcol - // WARNING: This means that very likely an edit for some other family - // was incorrectly stored into the store for this one. Throw an exception, - // because this might lead to data corruption. - throw new IOException( - "ScanWildcardColumnTracker.checkColumn ran into a column actually " + - "smaller than the previous column: " + - Bytes.toStringBinary(CellUtil.cloneQualifier(cell))); - } - - private void resetCell(Cell columnCell) { - this.columnCell = columnCell; - currentCount = 0; - } - - /** - * Check whether this version should be retained. - * There are 4 variables considered: - * If this version is past max versions -> skip it - * If this kv has expired or was deleted, check min versions - * to decide whther to skip it or not. - * - * Increase the version counter unless this is a delete - */ - private MatchCode checkVersion(byte type, long timestamp) { - if (!CellUtil.isDelete(type)) { - currentCount++; - } - if (currentCount > maxVersions) { - return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; // skip to next col - } - // keep the KV if required by minversions or it is not expired, yet - if (currentCount <= minVersions || !isExpired(timestamp)) { - setTSAndType(timestamp, type); - return ScanQueryMatcher.MatchCode.INCLUDE; - } else { - return MatchCode.SEEK_NEXT_COL; - } - - } - - @Override - public void reset() { - columnCell = null; - resetTSAndType(); - } - - private void resetTSAndType() { - latestTSOfCurrentColumn = HConstants.LATEST_TIMESTAMP; - latestTypeOfCurrentColumn = 0; - } - - private void setTSAndType(long timestamp, byte type) { - latestTSOfCurrentColumn = timestamp; - latestTypeOfCurrentColumn = type; - } - - private boolean sameAsPreviousTSAndType(long timestamp, byte type) { - return timestamp == latestTSOfCurrentColumn && type == latestTypeOfCurrentColumn; - } - - private boolean isExpired(long timestamp) { - return timestamp < oldestStamp; - } - - /** - * Used by matcher and scan/get to get a hint of the next column - * to seek to after checkColumn() returns SKIP. Returns the next interesting - * column we want, or NULL there is none (wildcard scanner). - * - * @return The column count. - */ - public ColumnCount getColumnHint() { - return null; - } - - /** - * We can never know a-priori if we are done, so always return false. - * @return false - */ - @Override - public boolean done() { - return false; - } - - @Override - public MatchCode getNextRowOrNextColumn(Cell cell) { - return MatchCode.SEEK_NEXT_COL; - } - - public boolean isDone(long timestamp) { - return minVersions <= 0 && isExpired(timestamp); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 5dec59a..853a4cf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java index 1582237..dca7388 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java @@ -18,6 +18,12 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.io.DataInput; +import java.io.IOException; +import java.util.Map; +import java.util.SortedSet; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -43,12 +49,6 @@ import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; -import java.io.DataInput; -import java.io.IOException; -import java.util.Map; -import java.util.SortedSet; -import java.util.concurrent.atomic.AtomicInteger; - /** * Reader for a StoreFile. */ @@ -114,46 +114,22 @@ public class StoreFileReader { } /** - * Uses {@link #getStoreFileScanner(boolean, boolean, boolean, long, long)} by setting - * {@code isCompaction} to false, {@code readPt} to 0 and {@code scannerOrder} to 0. - * Do not use this overload if using this scanner for compactions. - * - * @see #getStoreFileScanner(boolean, boolean, boolean, long, long) - */ - public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread) { - // 0 is passed as readpoint because this method is only used by test - // where StoreFile is directly operated upon - return getStoreFileScanner(cacheBlocks, pread, false, 0, 0); - } - - /** - * Uses {@link #getStoreFileScanner(boolean, boolean, boolean, long, long)} by setting - * {@code scannerOrder} to 0. - * - * @see #getStoreFileScanner(boolean, boolean, boolean, long, long) - */ - public StoreFileScanner getStoreFileScanner( - boolean cacheBlocks, boolean pread, boolean isCompaction, long readPt) { - return getStoreFileScanner(cacheBlocks, pread, isCompaction, readPt, 0); - } - - /** * Get a scanner to scan over this StoreFile. - * * @param cacheBlocks should this scanner cache blocks? * @param pread use pread (for highly concurrent small readers) * @param isCompaction is scanner being used for compaction? * @param scannerOrder Order of this scanner relative to other scanners. See - * {@link KeyValueScanner#getScannerOrder()}. + * {@link KeyValueScanner#getScannerOrder()}. + * @param canOptimizeForNonNullColumn {@code true} if we can make sure there is no null column, + * otherwise {@code false}. This is a hint for optimization. * @return a scanner */ - public StoreFileScanner getStoreFileScanner( - boolean cacheBlocks, boolean pread, boolean isCompaction, long readPt, long scannerOrder) { + public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread, + boolean isCompaction, long readPt, long scannerOrder, boolean canOptimizeForNonNullColumn) { // Increment the ref count refCount.incrementAndGet(); - return new StoreFileScanner( - this, getScanner(cacheBlocks, pread, isCompaction), !isCompaction, reader.hasMVCCInfo(), - readPt, scannerOrder); + return new StoreFileScanner(this, getScanner(cacheBlocks, pread, isCompaction), !isCompaction, + reader.hasMVCCInfo(), readPt, scannerOrder, canOptimizeForNonNullColumn); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/b35cf8f9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index 4955ffe..ac55d8c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; import org.apache.hadoop.hbase.util.Counter; /** @@ -54,49 +55,41 @@ public class StoreFileScanner implements KeyValueScanner { private boolean delayedReseek; private Cell delayedSeekKV; - private boolean enforceMVCC = false; - private boolean hasMVCCInfo = false; + private final boolean enforceMVCC; + private final boolean hasMVCCInfo; // A flag represents whether could stop skipping KeyValues for MVCC // if have encountered the next row. Only used for reversed scan private boolean stopSkippingKVsIfNextRow = false; private static Counter seekCount; - private ScanQueryMatcher matcher; + private final boolean canOptimizeForNonNullColumn; - private long readPt; + private final long readPt; // Order of this scanner relative to other scanners when duplicate key-value is found. // Higher values means scanner has newer data. - private long scannerOrder; + private final long scannerOrder; /** * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner} * @param useMVCC If true, scanner will filter out updates with MVCC larger than {@code readPt}. * @param readPt MVCC value to use to filter out the updates newer than this scanner. * @param hasMVCC Set to true if underlying store file reader has MVCC info. + * @param scannerOrder Order of the scanner relative to other scanners. See + * {@link KeyValueScanner#getScannerOrder()}. + * @param canOptimizeForNonNullColumn {@code true} if we can make sure there is no null column, + * otherwise {@code false}. This is a hint for optimization. */ public StoreFileScanner(StoreFileReader reader, HFileScanner hfs, boolean useMVCC, - boolean hasMVCC, long readPt) { - this (reader, hfs, useMVCC, hasMVCC, readPt, 0); - } - - /** - * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner} - * @param useMVCC If true, scanner will filter out updates with MVCC larger than {@code readPt}. - * @param readPt MVCC value to use to filter out the updates newer than this scanner. - * @param hasMVCC Set to true if underlying store file reader has MVCC info. - * @param scannerOrder Order of the scanner relative to other scanners. - * See {@link KeyValueScanner#getScannerOrder()}. - */ - public StoreFileScanner(StoreFileReader reader, HFileScanner hfs, boolean useMVCC, - boolean hasMVCC, long readPt, long scannerOrder) { + boolean hasMVCC, long readPt, long scannerOrder, boolean canOptimizeForNonNullColumn) { this.readPt = readPt; this.reader = reader; this.hfs = hfs; this.enforceMVCC = useMVCC; this.hasMVCCInfo = hasMVCC; this.scannerOrder = scannerOrder; + this.canOptimizeForNonNullColumn = canOptimizeForNonNullColumn; } boolean isPrimaryReplica() { @@ -126,24 +119,20 @@ public class StoreFileScanner implements KeyValueScanner { } /** - * Return an array of scanners corresponding to the given set of store files, - * And set the ScanQueryMatcher for each store file scanner for further - * optimization + * Return an array of scanners corresponding to the given set of store files, And set the + * ScanQueryMatcher for each store file scanner for further optimization */ - public static List<StoreFileScanner> getScannersForStoreFiles( - Collection<StoreFile> files, boolean cacheBlocks, boolean usePread, - boolean isCompaction, boolean canUseDrop, + public static List<StoreFileScanner> getScannersForStoreFiles(Collection<StoreFile> files, + boolean cacheBlocks, boolean usePread, boolean isCompaction, boolean canUseDrop, ScanQueryMatcher matcher, long readPt, boolean isPrimaryReplica) throws IOException { - List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>( - files.size()); + List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(files.size()); List<StoreFile> sorted_files = new ArrayList<>(files); Collections.sort(sorted_files, StoreFile.Comparators.SEQ_ID); for (int i = 0; i < sorted_files.size(); i++) { StoreFileReader r = sorted_files.get(i).createReader(); r.setReplicaStoreFile(isPrimaryReplica); - StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread, - isCompaction, readPt, i); - scanner.setScanQueryMatcher(matcher); + StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread, isCompaction, readPt, + i, matcher != null ? !matcher.hasNullColumnInQuery() : false); scanners.add(scanner); } return scanners; @@ -360,12 +349,12 @@ public class StoreFileScanner implements KeyValueScanner { // check ROWCOL Bloom filter first. if (reader.getBloomFilterType() == BloomType.ROWCOL) { haveToSeek = reader.passesGeneralRowColBloomFilter(kv); - } else if (this.matcher != null && !matcher.hasNullColumnInQuery() && - ((CellUtil.isDeleteFamily(kv) || CellUtil.isDeleteFamilyVersion(kv)))) { + } else if (canOptimizeForNonNullColumn + && ((CellUtil.isDeleteFamily(kv) || CellUtil.isDeleteFamilyVersion(kv)))) { // if there is no such delete family kv in the store file, // then no need to seek. - haveToSeek = reader.passesDeleteFamilyBloomFilter(kv.getRowArray(), - kv.getRowOffset(), kv.getRowLength()); + haveToSeek = reader.passesDeleteFamilyBloomFilter(kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength()); } } @@ -434,10 +423,6 @@ public class StoreFileScanner implements KeyValueScanner { } } - public void setScanQueryMatcher(ScanQueryMatcher matcher) { - this.matcher = matcher; - } - @Override public boolean isFileScanner() { return true;
