PHOENIX-3383 Comparison between descending row keys used in RVC is reverse

Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/aee568be
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/aee568be
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/aee568be

Branch: refs/heads/master
Commit: aee568beb02cdf983bb10889902c338ea016e6c9
Parents: d35a751
Author: James Taylor <jamestay...@apache.org>
Authored: Thu Jul 5 21:38:28 2018 -0700
Committer: James Taylor <jamestay...@apache.org>
Committed: Wed Jul 11 07:28:47 2018 -0700

----------------------------------------------------------------------
 .../org/apache/phoenix/end2end/QueryMoreIT.java |   12 +-
 .../org/apache/phoenix/compile/KeyPart.java     |    2 -
 .../org/apache/phoenix/compile/ScanRanges.java  |   77 +-
 .../apache/phoenix/compile/WhereOptimizer.java  | 1304 +++++++++++++-----
 .../PhoenixTxIndexMutationGenerator.java        |    2 +-
 .../expression/function/FunctionExpression.java |   10 +-
 .../expression/function/InvertFunction.java     |   19 +-
 .../expression/function/PrefixFunction.java     |    6 +-
 .../expression/function/RTrimFunction.java      |    6 +-
 .../function/RoundDateExpression.java           |   22 +-
 .../function/RoundDecimalExpression.java        |    7 +-
 .../phoenix/iterate/BaseResultIterators.java    |    4 +-
 .../apache/phoenix/iterate/ExplainTable.java    |   10 -
 .../java/org/apache/phoenix/query/KeyRange.java |   28 +-
 .../org/apache/phoenix/schema/RowKeySchema.java |   78 ++
 .../phoenix/compile/QueryCompilerTest.java      |    2 +-
 .../phoenix/compile/QueryOptimizerTest.java     |    5 +-
 .../TenantSpecificViewIndexCompileTest.java     |    8 +-
 .../phoenix/compile/WhereOptimizerTest.java     |  359 ++++-
 .../RoundFloorCeilExpressionsTest.java          |   59 +-
 .../apache/phoenix/query/KeyRangeClipTest.java  |    2 +-
 .../org/apache/phoenix/query/QueryPlanTest.java |    8 +-
 .../apache/phoenix/schema/RowKeySchemaTest.java |   48 +
 23 files changed, 1567 insertions(+), 511 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
index 9109c12..04272fa 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
@@ -372,9 +372,6 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
         }
     }
     
-    // FIXME: this repros PHOENIX-3382, but turned up two more issues:
-    // 1) PHOENIX-3383 Comparison between descending row keys used in RVC is 
reverse
-    // 2) PHOENIX-3384 Optimize RVC expressions for non leading row key columns
     @Test
     public void testRVCOnDescWithLeadingPKEquality() throws Exception {
         final Connection conn = DriverManager.getConnection(getUrl());
@@ -398,14 +395,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
         conn.createStatement().execute("UPSERT INTO " + fullTableName + " 
VALUES ('org1',1,'02')");
         conn.commit();
 
-        // FIXME: PHOENIX-3383
-        // This comparison is really backwards: it should be (score, 
entity_id) < (2, '04'),
-        // but because we're matching a descending key, our comparison has to 
be switched.
         try (Statement stmt = conn.createStatement()) {
             final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" 
+ 
                     "FROM " + fullTableName + "\n" + 
                     "WHERE organization_id = 'org1'\n" + 
-                    "AND (score, entity_id) > (2, '04')\n" + 
+                    "AND (score, entity_id) < (2, '04')\n" + 
                     "ORDER BY score DESC, entity_id DESC\n" + 
                     "LIMIT 3");
             assertTrue(rs.next());
@@ -416,13 +410,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
             assertEquals(1.0, rs.getDouble(2), 0.001);
             assertFalse(rs.next());
         }
-        // FIXME: PHOENIX-3384
-        // It should not be necessary to specify organization_id in this query
         try (Statement stmt = conn.createStatement()) {
             final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" 
+ 
                     "FROM " + fullTableName + "\n" + 
                     "WHERE organization_id = 'org1'\n" + 
-                    "AND (organization_id, score, entity_id) > ('org1', 2, 
'04')\n" + 
+                    "AND (organization_id, score, entity_id) < ('org1', 2, 
'04')\n" + 
                     "ORDER BY score DESC, entity_id DESC\n" + 
                     "LIMIT 3");
             assertTrue(rs.next());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
index 55385f2..6cf9938 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
@@ -33,8 +33,6 @@ import org.apache.phoenix.schema.PTable;
  * between a built-in function and the setting of the scan key
  * during query compilation.
  * 
- * 
- * @since 0.12
  */
 public interface KeyPart {
     /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index 019f15d..1732fce 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.filter.SkipScanFilter;
@@ -45,7 +44,6 @@ import org.apache.phoenix.schema.types.PDataType.PDataCodec;
 import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.ScanUtil;
-import org.apache.phoenix.util.ScanUtil.BytesComparator;
 import org.apache.phoenix.util.SchemaUtil;
 
 import com.google.common.base.Throwables;
@@ -56,30 +54,30 @@ import com.google.common.collect.Lists;
 public class ScanRanges {
     private static final List<List<KeyRange>> EVERYTHING_RANGES = 
Collections.<List<KeyRange>>emptyList();
     private static final List<List<KeyRange>> NOTHING_RANGES = 
Collections.<List<KeyRange>>singletonList(Collections.<KeyRange>singletonList(KeyRange.EMPTY_RANGE));
-    public static final ScanRanges EVERYTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, 
KeyRange.EVERYTHING_RANGE, KeyRange.EVERYTHING_RANGE, false, false, null, null);
-    public static final ScanRanges NOTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, 
KeyRange.EMPTY_RANGE, KeyRange.EMPTY_RANGE, false, false, null, null);
+    public static final ScanRanges EVERYTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, 
KeyRange.EVERYTHING_RANGE, false, false, null, null);
+    public static final ScanRanges NOTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, 
KeyRange.EMPTY_RANGE, false, false, null, null);
     private static final Scan HAS_INTERSECTION = new Scan();
 
     public static ScanRanges createPointLookup(List<KeyRange> keys) {
-        return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, 
Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, 
KeyRange.EVERYTHING_RANGE, null, true, -1);
+        return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, 
Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, 
-1);
     }
     
     // For testing
     public static ScanRanges createSingleSpan(RowKeySchema schema, 
List<List<KeyRange>> ranges) {
-        return create(schema, ranges, 
ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, null, 
true, -1);
+        return create(schema, ranges, 
ScanUtil.getDefaultSlotSpans(ranges.size()), null, true, -1);
     }
     
     // For testing
     public static ScanRanges createSingleSpan(RowKeySchema schema, 
List<List<KeyRange>> ranges, Integer nBuckets, boolean useSkipSan) {
-        return create(schema, ranges, 
ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, 
nBuckets, useSkipSan, -1);
+        return create(schema, ranges, 
ScanUtil.getDefaultSlotSpans(ranges.size()), nBuckets, useSkipSan, -1);
     }
 
-    public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> 
ranges, int[] slotSpan, KeyRange minMaxRange, Integer nBuckets, boolean 
useSkipScan, int rowTimestampColIndex) {
+    public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> 
ranges, int[] slotSpan, Integer nBuckets, boolean useSkipScan, int 
rowTimestampColIndex) {
         int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
         int nSlots = ranges.size();
-        if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
+        if (nSlots == offset) {
             return EVERYTHING;
-        } else if (minMaxRange == KeyRange.EMPTY_RANGE || (nSlots == 1 + 
offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == 
KeyRange.EMPTY_RANGE)) {
+        } else if ((nSlots == 1 + offset && ranges.get(offset).size() == 1 && 
ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
             return NOTHING;
         }
         TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, 
schema, rowTimestampColIndex);
@@ -88,22 +86,9 @@ public class ScanRanges {
             // TODO: consider keeping original to use for serialization as it 
would be smaller?
             List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, 
schema, nBuckets);
             List<KeyRange> keyRanges = 
Lists.newArrayListWithExpectedSize(keys.size());
-            KeyRange unsaltedMinMaxRange = minMaxRange;
-            if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
-                unsaltedMinMaxRange = KeyRange.getKeyRange(
-                        stripPrefix(minMaxRange.getLowerRange(),offset),
-                        minMaxRange.lowerUnbound(), 
-                        stripPrefix(minMaxRange.getUpperRange(),offset), 
-                        minMaxRange.upperUnbound());
-            }
             // We have full keys here, so use field from our varbinary schema
-            BytesComparator comparator = 
ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
             for (byte[] key : keys) {
-                // Filter now based on unsalted minMaxRange and ignore the 
point key salt byte
-                if ( unsaltedMinMaxRange.compareLowerToUpperBound(key, offset, 
key.length-offset, true, comparator) <= 0 &&
-                     unsaltedMinMaxRange.compareUpperToLowerBound(key, offset, 
key.length-offset, true, comparator) >= 0) {
-                    keyRanges.add(KeyRange.getKeyRange(key));
-                }
+                keyRanges.add(KeyRange.getKeyRange(key));
             }
             // while doing a point look up if after intersecting with the 
MinMaxrange there are
             // no more keyranges left then just return
@@ -151,17 +136,11 @@ public class ScanRanges {
             }
             scanRange = KeyRange.getKeyRange(minKey, maxKey);
         }
-        if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
-            // Intersect using modified min/max range, but keep original range 
to ensure it
-            // can still be decomposed into it's parts
-            KeyRange inclusiveExclusiveMinMaxRange = 
ScanUtil.convertToInclusiveExclusiveRange(minMaxRange, schema, new 
ImmutableBytesWritable());
-            scanRange = scanRange.intersect(inclusiveExclusiveMinMaxRange);
-        }
-        
+
         if (scanRange == KeyRange.EMPTY_RANGE) {
             return NOTHING;
         }
-        return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, 
minMaxRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange);
+        return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, 
useSkipScan, isPointLookup, nBuckets, rowTimestampRange);
     }
 
     private SkipScanFilter filter;
@@ -172,15 +151,13 @@ public class ScanRanges {
     private final boolean isSalted;
     private final boolean useSkipScanFilter;
     private final KeyRange scanRange;
-    private final KeyRange minMaxRange;
     private final TimeRange rowTimestampRange;
 
-    private ScanRanges (RowKeySchema schema, int[] slotSpan, 
List<List<KeyRange>> ranges, KeyRange scanRange, KeyRange minMaxRange, boolean 
useSkipScanFilter, boolean isPointLookup, Integer bucketNum, TimeRange 
rowTimestampRange) {
+    private ScanRanges (RowKeySchema schema, int[] slotSpan, 
List<List<KeyRange>> ranges, KeyRange scanRange, boolean useSkipScanFilter, 
boolean isPointLookup, Integer bucketNum, TimeRange rowTimestampRange) {
         this.isPointLookup = isPointLookup;
         this.isSalted = bucketNum != null;
         this.useSkipScanFilter = useSkipScanFilter;
         this.scanRange = scanRange;
-        this.minMaxRange = minMaxRange;
         this.rowTimestampRange = rowTimestampRange;
         
         if (isSalted && !isPointLookup) {
@@ -199,14 +176,6 @@ public class ScanRanges {
         }
     }
     
-    /**
-     * Get the minMaxRange that is applied in addition to the scan range.
-     * Only used by the ExplainTable to generate the explain plan.
-     */
-    public KeyRange getMinMaxRange() {
-        return minMaxRange;
-    }
-    
     public void initializeScan(Scan scan) {
         scan.setStartRow(scanRange.getLowerRange());
         scan.setStopRow(scanRange.getUpperRange());
@@ -580,29 +549,9 @@ public class ScanRanges {
     }
 
     public int getBoundPkColumnCount() {
-        return Math.max(getBoundPkSpan(ranges, slotSpan), 
getBoundMinMaxSlotCount());
+        return getBoundPkSpan(ranges, slotSpan);
     }
 
-    private int getBoundMinMaxSlotCount() {
-        if (minMaxRange == KeyRange.EMPTY_RANGE || minMaxRange == 
KeyRange.EVERYTHING_RANGE) {
-            return 0;
-        }
-        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
-        // We don't track how many slots are bound for the minMaxRange, so we 
need
-        // to traverse the upper and lower range key and count the slots.
-        int lowerCount = 0;
-        int maxOffset = schema.iterator(minMaxRange.getLowerRange(), ptr);
-        for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, 
maxOffset)); pos++) {
-            lowerCount++;
-        }
-        int upperCount = 0;
-        maxOffset = schema.iterator(minMaxRange.getUpperRange(), ptr);
-        for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, 
maxOffset)); pos++) {
-            upperCount++;
-        }
-        return Math.max(lowerCount, upperCount);
-    }
-    
     public int getBoundSlotCount() {
         int count = 0;
         boolean hasUnbound = false;

Reply via email to