This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new c1aa9f1 PHOENIX-6458 Using global indexes for queries with uncovered
columns (#1256)
c1aa9f1 is described below
commit c1aa9f1b18ea8be94aa96bce04fd57e2ec715aaf
Author: kadirozde <[email protected]>
AuthorDate: Thu Feb 24 14:56:24 2022 -0800
PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256)
---
.../end2end/index/GlobalIndexCheckerIT.java | 122 +++++++++++++++++++++
.../end2end/index/GlobalIndexOptimizationIT.java | 78 ++++++-------
.../apache/phoenix/compile/ExpressionCompiler.java | 11 +-
.../org/apache/phoenix/compile/FromCompiler.java | 14 ++-
.../org/apache/phoenix/compile/JoinCompiler.java | 11 +-
.../apache/phoenix/compile/ProjectionCompiler.java | 17 ++-
.../apache/phoenix/compile/StatementContext.java | 9 ++
.../phoenix/compile/TupleProjectionCompiler.java | 23 ++--
.../org/apache/phoenix/compile/WhereCompiler.java | 9 +-
.../coprocessor/BaseScannerRegionObserver.java | 7 ++
.../GroupedAggregateRegionObserver.java | 12 +-
.../org/apache/phoenix/execute/BaseQueryPlan.java | 27 +++--
.../phoenix/iterate/BaseResultIterators.java | 7 +-
.../org/apache/phoenix/iterate/ExplainTable.java | 6 +-
.../iterate/NonAggregateRegionScannerFactory.java | 24 ++--
.../phoenix/iterate/RegionScannerFactory.java | 23 ++--
.../apache/phoenix/optimize/QueryOptimizer.java | 1 +
...xDataColumnRef.java => IndexDataColumnRef.java} | 41 +++++--
.../java/org/apache/phoenix/schema/TableRef.java | 32 ++++--
.../java/org/apache/phoenix/util/IndexUtil.java | 62 +++++++++--
.../java/org/apache/phoenix/util/ScanUtil.java | 11 ++
21 files changed, 403 insertions(+), 144 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
index cb01149..02c57e2 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
@@ -306,6 +306,44 @@ public class GlobalIndexCheckerIT extends BaseTest {
assertEquals("ae", rs.getString(1));
assertEquals("efg", rs.getString(2));
assertFalse(rs.next());
+ conn.createStatement().execute("DROP INDEX " + indexTableName + "
on " +
+ dataTableName);
+ // Run the previous test on an uncovered global index
+ indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName +
" on " +
+ dataTableName + " (PHOENIX_ROW_TIMESTAMP())" +
+ (async ? "ASYNC" : "")+ this.indexDDLOptions);
+ if (async) {
+ // Run the index MR job to rebuild the index and verify that
index is built correctly
+ IndexToolIT.runIndexTool(false, null, dataTableName,
+ indexTableName, null, 0,
IndexTool.IndexVerifyType.AFTER);
+ }
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
query);
+
+ // Verify that we will read from the index table with the index
hint
+ query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ " +
+ "val1, val2, PHOENIX_ROW_TIMESTAMP() from " +
dataTableName + " WHERE " +
+ "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString()
+ "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')";
+
+ assertExplainPlan(conn, query, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("ab", rs.getString(1));
+ assertEquals("abc", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("bc", rs.getString(1));
+ assertEquals("bcd", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("bc", rs.getString(1));
+ assertEquals("ccc", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("de", rs.getString(1));
+ assertEquals("def", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ae", rs.getString(1));
+ assertEquals("efg", rs.getString(2));
+ assertFalse(rs.next());
}
}
@@ -430,6 +468,90 @@ public class GlobalIndexCheckerIT extends BaseTest {
}
}
+ private void assertIndexTableNotSelected(Connection conn, String
dataTableName, String indexTableName, String sql)
+ throws Exception {
+ try {
+ assertExplainPlan(conn, sql, dataTableName, indexTableName);
+ throw new AssertionError("The index table should not be selected
without an index hint");
+ } catch (AssertionError error){
+ //expected
+ }
+ }
+
+ @Test
+ public void testUncoveredGlobalIndex() throws Exception {
+ if (async) {
+ return;
+ }
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName); // with two rows ('a', 'ab', 'abc',
'abcd') and ('b', 'bc', 'bcd', 'bcde')
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName +
" on " +
+ dataTableName + " (val1) include (val2)" +
this.indexDDLOptions);
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT val3 from " + dataTableName + " WHERE val1 = 'bc'
AND (val2 = 'bcd' OR val3 ='bcde')");
+
+ //Verify that with index hint, we will read from the index table
even though val3 is not included by the index table
+ String selectSql = "SELECT /*+ INDEX(" + dataTableName + " " +
indexTableName + ")*/ val3 from "
+ + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR
val3 ='bcde')";
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("bcde", rs.getString(1));
+ assertFalse(rs.next());
+ conn.createStatement().execute("DROP INDEX " + indexTableName + "
on " + dataTableName);
+ // Create an index does not include any columns
+ indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName +
" on " +
+ dataTableName + " (val1)" + this.indexDDLOptions);
+ conn.commit();
+
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT id from " + dataTableName + " WHERE val1 = 'bc'
AND (val2 = 'bcd' OR val3 ='bcde')");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " +
indexTableName + ")*/ id from " + dataTableName + " WHERE val1 = 'bc' AND (val2
= 'bcd' OR val3 ='bcde')";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertFalse(rs.next());
+
+ // Add another row and run a group by query where the uncovered
index should be used
+ conn.createStatement().execute("upsert into " + dataTableName + "
(id, val1, val2, val3) values ('c', 'ab','cde', 'cdef')");
+ conn.commit();
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT count(*) from " + dataTableName + " where val1 >
'0' GROUP BY val1");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " +
indexTableName + ")*/ count(*) from " + dataTableName + " where val1 > '0'
GROUP BY val1";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+ // Run an order by query where the uncovered index should be used
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT val3 from " + dataTableName + " where val1 > '0'
ORDER BY val1");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " +
indexTableName + ")*/ val3 from " + dataTableName + " where val1 > '0' ORDER BY
val1";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("abcd", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("cdef", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("bcde", rs.getString(1));
+ assertFalse(rs.next());
+ }
+ }
+
@Test
public void testSimulateConcurrentUpdates() throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl())) {
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
index 96e83f5..99f5997 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
@@ -230,17 +230,12 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where v1<='z' and
k3 > 1 order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName
+ "\n" +
- " SERVER FILTER BY K3 > 1\n" +
- " SERVER SORTED BY \\[" + dataTableName + ".V1, " +
dataTableName + ".T_ID\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName +
".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN
\\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName
+ " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY AND \"K3\" > 1";
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -265,17 +260,15 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ t_id, V1, k3 from " + dataTableFullName + " where v1 <='z' group by
v1,t_id, k3";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
-
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName
+ "\n" +
- " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" +
dataTableName + ".V1, " + dataTableName + ".T_ID, " + dataTableName +
".K3\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" +
dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName +
".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName
+ " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\",
\"T_ID\", \"K3\"]\n"+
+ "CLIENT MERGE SORT";
+
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -299,16 +292,13 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ v1,sum(k3) from " + dataTableFullName + " where v1 <='z' group by v1
order by v1";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName
+ "\n" +
- " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" +
dataTableName + ".V1\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" +
dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName +
".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName
+ " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS
BY [\"V1\"]";
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -340,12 +330,10 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN "+
query);
String actual = QueryUtil.getExplainPlan(rs);
- String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
dataTableName + " \\['tid1'\\]\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
indexTableName + " \\['tid1','a'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" +
dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN
\\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
- assertTrue("Expected:\n" + expected + "\ndid not match\n" +
actual, Pattern.matches(expected, actual));
+ String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +
indexTableName + " ['tid1','a']\n" +
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY";
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual,
actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -393,15 +381,15 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
* This inner "_IDX_" + dataTableName use skipScan, and all the
whereExpressions are already in SkipScanFilter,
* so there is no other RowKeyComparisonFilter needed.
*/
+
String actual = QueryUtil.getExplainPlan(rs);
String expected =
- "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableName +
"\n" +
- " SERVER FILTER BY V1 = 'a'\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER
_IDX_" + dataTableName + " \\[" + Short.MIN_VALUE + ",1\\] - \\[" +
Short.MIN_VALUE + ",2\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + viewName +
".T_ID\", \"" + viewName + ".K1\", \"" + viewName + ".K2\"\\) IN
\\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
- assertTrue("Expected:\n" + expected + "\ndid not match\n" +
actual, Pattern.matches(expected,actual));
+ "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" +
dataTableName +
+ " [" + Short.MIN_VALUE + ",1] - [" +
Short.MIN_VALUE + ",2]\n" +
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY";
+
+ assertEquals(expected,actual);
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -409,7 +397,7 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
assertEquals(2, rs.getInt("k1"));
assertEquals(4, rs.getInt("k2"));
assertEquals(2, rs.getInt("k3"));
- assertEquals("a", rs.getString("v1"));
+ assertEquals("a", rs.getString(5)); //TODO use name v1 instead of
position 5, see PHOENIX-6644
assertFalse(rs.next());
} finally {
conn1.close();
@@ -474,7 +462,7 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
assertFalse(rs.next());
// No where clause
- query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id";
+ query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName +
" order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(
@@ -511,7 +499,7 @@ public class GlobalIndexOptimizationIT extends
ParallelStatsDisabledIT {
assertFalse(rs.next());
// No where clause in index scan
- query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName
+ ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order
by V1,t_id";
+ query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName +
" where k3 > 1 order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
index d1b44df..986cdf3 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
@@ -112,7 +112,7 @@ import
org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.DelegateDatum;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PTable;
@@ -140,6 +140,8 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
+
public class ExpressionCompiler extends
UnsupportedAllParseNodeVisitor<Expression> {
private boolean isAggregate;
@@ -371,9 +373,12 @@ public class ExpressionCompiler extends
UnsupportedAllParseNodeVisitor<Expressio
// Rather than not use a local index when a column not contained
by it is referenced, we
// join back to the data table in our coprocessor since this is a
relatively cheap
// operation given that we know the join is local.
- if (context.getCurrentTable().getTable().getIndexType() ==
IndexType.LOCAL) {
+ if (context.getCurrentTable().getTable().getIndexType() ==
IndexType.LOCAL
+ || isHintedGlobalIndex(context.getCurrentTable())) {
try {
- return new LocalIndexDataColumnRef(context,
context.getCurrentTable(), node.getName());
+ context.setUncoveredIndex(true);
+ return new IndexDataColumnRef(context,
context.getCurrentTable(),
+ node.getName());
} catch (ColumnFamilyNotFoundException c) {
throw e;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 197a0a6..88d80aa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -103,6 +103,7 @@ import
org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import static
org.apache.phoenix.monitoring.MetricType.NUM_METADATA_LOOKUP_FAILURES;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
/**
* Validates FROM clause and builds a ColumnResolver for resolving column
references
@@ -1157,13 +1158,14 @@ public class FromCompiler {
}
private static class ProjectedTableColumnResolver extends
MultiTableColumnResolver {
- private final boolean isLocalIndex;
+ private final boolean isIndex;
private final List<TableRef> theTableRefs;
private final Map<ColumnRef, Integer> columnRefMap;
private ProjectedTableColumnResolver(PTable projectedTable,
PhoenixConnection conn, Map<String, UDFParseNode> udfParseNodes) throws
SQLException {
super(conn, 0, udfParseNodes, null);
Preconditions.checkArgument(projectedTable.getType() ==
PTableType.PROJECTED);
- this.isLocalIndex = projectedTable.getIndexType() ==
IndexType.LOCAL;
+ this.isIndex = projectedTable.getIndexType() == IndexType.LOCAL
+ || projectedTable.getIndexType() == IndexType.GLOBAL;
this.columnRefMap = new HashMap<ColumnRef, Integer>();
long ts = Long.MAX_VALUE;
for (int i = projectedTable.getBucketNum() == null ? 0 : 1; i <
projectedTable.getColumns().size(); i++) {
@@ -1201,9 +1203,11 @@ public class FromCompiler {
try {
colRef = super.resolveColumn(schemaName, tableName, colName);
} catch (ColumnNotFoundException e) {
- // This could be a ColumnRef for local index data column.
- TableRef tableRef = isLocalIndex ? super.getTables().get(0) :
super.resolveTable(schemaName, tableName);
- if (tableRef.getTable().getIndexType() == IndexType.LOCAL) {
+ // This could be a ColumnRef for index data column.
+ TableRef tableRef = isIndex ? super.getTables().get(0)
+ : super.resolveTable(schemaName, tableName);
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
TableRef parentTableRef = super.resolveTable(
tableRef.getTable().getSchemaName().getString(),
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index de49826..70345b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.compile;
import static
org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
import static
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -75,7 +76,7 @@ import org.apache.phoenix.parse.TableNodeVisitor;
import org.apache.phoenix.parse.TableWildcardParseNode;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PNameFactory;
@@ -1127,7 +1128,8 @@ public class JoinCompiler {
if (columnRef.getTableRef().equals(tableRef)
&& (!retainPKColumns ||
!SchemaUtil.isPKColumn(columnRef.getColumn()))) {
if (columnRef instanceof LocalIndexColumnRef) {
- sourceColumns.add(new
LocalIndexDataColumnRef(context, tableRef,
IndexUtil.getIndexColumnName(columnRef.getColumn())));
+ sourceColumns.add(new IndexDataColumnRef(context,
tableRef,
+
IndexUtil.getIndexColumnName(columnRef.getColumn())));
} else {
sourceColumns.add(columnRef);
}
@@ -1392,10 +1394,11 @@ public class JoinCompiler {
try {
columnRef = resolver.resolveColumn(node.getSchemaName(),
node.getTableName(), node.getName());
} catch (ColumnNotFoundException e) {
- // This could be a LocalIndexDataColumnRef. If so, the table
name must have
+ // This could be an IndexDataColumnRef. If so, the table name
must have
// been appended by the IndexStatementRewriter, and we can
convert it into.
TableRef tableRef =
resolver.resolveTable(node.getSchemaName(), node.getTableName());
- if (tableRef.getTable().getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
TableRef parentTableRef = FromCompiler.getResolver(
NODE_FACTORY.namedTable(null,
TableName.create(tableRef.getTable()
.getSchemaName().getString(),
tableRef.getTable()
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index c13f383..f36f605 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.compile;
import static
org.apache.phoenix.query.QueryServices.WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB;
import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
@@ -69,9 +70,9 @@ import
org.apache.phoenix.schema.ArgumentTypeMismatchException;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.KeyValueSchema;
import org.apache.phoenix.schema.KeyValueSchema.KeyValueSchemaBuilder;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PDatum;
@@ -230,9 +231,11 @@ public class ProjectionCompiler {
indexColumn = index.getColumnForColumnName(indexColName);
ref = new ColumnRef(tableRef, indexColumn.getPosition());
} catch (ColumnNotFoundException e) {
- if (index.getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
- ref = new LocalIndexDataColumnRef(context, tableRef,
indexColName);
+ context.setUncoveredIndex(true);
+ ref = new IndexDataColumnRef(context, tableRef,
indexColName);
indexColumn = ref.getColumn();
} catch (ColumnFamilyNotFoundException c) {
throw e;
@@ -303,9 +306,11 @@ public class ProjectionCompiler {
ref = new ColumnRef(tableRef, indexColumn.getPosition());
indexColumnFamily = indexColumn.getFamilyName() == null ? null
: indexColumn.getFamilyName().getString();
} catch (ColumnNotFoundException e) {
- if (index.getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
- ref = new LocalIndexDataColumnRef(context, tableRef,
indexColName);
+ context.setUncoveredIndex(true);
+ ref = new IndexDataColumnRef(context, tableRef,
indexColName);
indexColumn = ref.getColumn();
indexColumnFamily =
indexColumn.getFamilyName() == null ? null
@@ -702,7 +707,7 @@ public class ProjectionCompiler {
PColumn col = expression.getColumn();
// hack'ish... For covered columns with local
indexes we defer to the server.
if (col instanceof ProjectedColumn &&
((ProjectedColumn) col)
- .getSourceColumnRef() instanceof
LocalIndexDataColumnRef) {
+ .getSourceColumnRef() instanceof
IndexDataColumnRef) {
return null;
}
PTable table =
context.getCurrentTable().getTable();
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 2abd546..d6cc20b 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -87,6 +87,7 @@ public class StatementContext {
private final OverAllQueryMetrics overAllQueryMetrics;
private QueryLogger queryLogger;
private boolean isClientSideUpsertSelect;
+ private boolean isUncoveredIndex;
public StatementContext(PhoenixStatement statement) {
this(statement, new Scan());
@@ -330,6 +331,14 @@ public class StatementContext {
this.isClientSideUpsertSelect = isClientSideUpsertSelect;
}
+ public boolean isUncoveredIndex() {
+ return isUncoveredIndex;
+ }
+
+ public void setUncoveredIndex(boolean isUncoveredIndex) {
+ this.isUncoveredIndex = isUncoveredIndex;
+ }
+
/*
* setRetryingPersistentCache can be used to override the
USE_PERSISTENT_CACHE hint and disable the use of the
* persistent cache for a specific cache ID. This can be used to retry
queries that failed when using the persistent
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 2a67d8d..9174f50 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.compile;
import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
import static
org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -41,13 +42,12 @@ import org.apache.phoenix.parse.WildcardParseNode;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.EncodedCQCounter;
-import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ProjectedColumn;
@@ -154,14 +154,18 @@ public class TupleProjectionCompiler {
EncodedColumnsUtil.setColumns(column, table, context.getScan());
}
}
- // add LocalIndexDataColumnRef
+ // add IndexDataColumnRef
position = projectedColumns.size();
- for (LocalIndexDataColumnRef sourceColumnRef :
visitor.localIndexColumnRefSet) {
+ for (IndexDataColumnRef sourceColumnRef : visitor.indexColumnRefSet) {
PColumn column = new
ProjectedColumn(sourceColumnRef.getColumn().getName(),
sourceColumnRef.getColumn().getFamilyName(), position++,
sourceColumnRef.getColumn().isNullable(), sourceColumnRef,
sourceColumnRef.getColumn().getColumnQualifierBytes());
projectedColumns.add(column);
}
+ if (!visitor.indexColumnRefSet.isEmpty()
+ && tableRef.isHinted()) {
+ context.setUncoveredIndex(true);
+ }
return PTableImpl.builderWithColumns(table, projectedColumns)
.setType(PTableType.PROJECTED)
.setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
@@ -230,12 +234,12 @@ public class TupleProjectionCompiler {
private static class ColumnRefVisitor extends
StatelessTraverseAllParseNodeVisitor {
private final StatementContext context;
private final LinkedHashSet<ColumnRef> nonPkColumnRefSet;
- private final LinkedHashSet<LocalIndexDataColumnRef>
localIndexColumnRefSet;
+ private final LinkedHashSet<IndexDataColumnRef> indexColumnRefSet;
private ColumnRefVisitor(StatementContext context) {
this.context = context;
this.nonPkColumnRefSet = new LinkedHashSet<ColumnRef>();
- this.localIndexColumnRefSet = new
LinkedHashSet<LocalIndexDataColumnRef>();
+ this.indexColumnRefSet = new LinkedHashSet<IndexDataColumnRef>();
}
@Override
@@ -247,9 +251,12 @@ public class TupleProjectionCompiler {
nonPkColumnRefSet.add(resolveColumn);
}
} catch (ColumnNotFoundException e) {
- if (context.getCurrentTable().getTable().getIndexType() ==
IndexType.LOCAL) {
+ if (context.getCurrentTable().getTable().getIndexType() ==
PTable.IndexType.LOCAL
+ || isHintedGlobalIndex(context.getCurrentTable())) {
try {
- localIndexColumnRefSet.add(new
LocalIndexDataColumnRef(context, context.getCurrentTable(), node.getName()));
+ context.setUncoveredIndex(true);
+ indexColumnRefSet.add(new IndexDataColumnRef(context,
+ context.getCurrentTable(), node.getName()));
} catch (ColumnFamilyNotFoundException c) {
throw e;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
index 9439a3c..1f6ab7f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
@@ -278,7 +278,9 @@ public class WhereCompiler {
if (LiteralExpression.isBooleanFalseOrNull(whereClause)) {
context.setScanRanges(ScanRanges.NOTHING);
- } else if (context.getCurrentTable().getTable().getIndexType() ==
IndexType.LOCAL) {
+ } else if (context.getCurrentTable().getTable().getIndexType() ==
IndexType.LOCAL
+ || (context.getCurrentTable().getTable().getIndexType() ==
IndexType.GLOBAL
+ && context.isUncoveredIndex())) {
if (whereClause != null &&
!ExpressionUtil.evaluatesToTrue(whereClause)) {
// pass any extra where as scan attribute so it can be
evaluated after all
// columns from the main CF have been merged in
@@ -291,11 +293,12 @@ public class WhereCompiler {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER,
stream.toByteArray());
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER,
stream.toByteArray());
// this is needed just for ExplainTable, since de-serializing
an expression does not restore
// its display properties, and that cannot be changed, due to
backwards compatibility
-
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR,
Bytes.toBytes(whereClause.toString()));
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR,
+ Bytes.toBytes(whereClause.toString()));
}
} else if (whereClause != null &&
!ExpressionUtil.evaluatesToTrue(whereClause)) {
Filter filter = null;
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 9a63e88..64e2dae 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -86,6 +86,7 @@ abstract public class BaseScannerRegionObserver extends
BaseRegionObserver {
public static final String GROUP_BY_LIMIT = "_GroupByLimit";
public static final String LOCAL_INDEX = "_LocalIndex";
public static final String LOCAL_INDEX_BUILD = "_LocalIndexBuild";
+ public static final String UNCOVERED_GLOBAL_INDEX =
"_UncoveredGlobalIndex";
public static final String INDEX_REBUILD_PAGING = "_IndexRebuildPaging";
// The number of index rows to be rebuild in one RPC call
public static final String INDEX_REBUILD_PAGE_ROWS =
"_IndexRebuildPageRows";
@@ -97,9 +98,15 @@ abstract public class BaseScannerRegionObserver extends
BaseRegionObserver {
"_IndexRebuildDisableLoggingVerifyType";
public static final String
INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE =
"_IndexRebuildDisableLoggingBeyondMaxLookbackAge";
+ @Deprecated
public static final String LOCAL_INDEX_FILTER = "_LocalIndexFilter";
+ @Deprecated
public static final String LOCAL_INDEX_LIMIT = "_LocalIndexLimit";
+ @Deprecated
public static final String LOCAL_INDEX_FILTER_STR = "_LocalIndexFilterStr";
+ public static final String INDEX_FILTER = "_IndexFilter";
+ public static final String INDEX_LIMIT = "_IndexLimit";
+ public static final String INDEX_FILTER_STR = "_IndexFilterStr";
/*
* Attribute to denote that the index maintainer has been serialized using
its proto-buf presentation.
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
index 00fd405..306f1fe 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
@@ -136,13 +136,8 @@ public class GroupedAggregateRegionObserver extends
BaseScannerRegionObserver {
.getEnvironment().getConfiguration(), em);
RegionScanner innerScanner = s;
- boolean useProto = false;
- byte[] localIndexBytes =
scan.getAttribute(LOCAL_INDEX_BUILD_PROTO);
- useProto = localIndexBytes != null;
- if (localIndexBytes == null) {
- localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
- }
- List<IndexMaintainer> indexMaintainers = localIndexBytes == null ?
null : IndexMaintainer.deserialize(localIndexBytes, useProto);
+ List<IndexMaintainer> indexMaintainers =
+ IndexUtil.deSerializeIndexMaintainersFromScan(scan);
TupleProjector tupleProjector = null;
byte[][] viewConstants = null;
ColumnReference[] dataColumns =
IndexUtil.deserializeDataTableColumnsToJoin(scan);
@@ -150,7 +145,8 @@ public class GroupedAggregateRegionObserver extends
BaseScannerRegionObserver {
final TupleProjector p =
TupleProjector.deserializeProjectorFromScan(scan);
final HashJoinInfo j =
HashJoinInfo.deserializeHashJoinFromScan(scan);
boolean useQualifierAsIndex =
EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan));
- if (ScanUtil.isLocalIndex(scan) || (j == null && p != null)) {
+ if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan)
+ || (j == null && p != null)) {
if (dataColumns != null) {
tupleProjector = IndexUtil.getTupleProjector(scan,
dataColumns);
viewConstants =
IndexUtil.deserializeViewConstantsFromScan(scan);
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index d9aacd7..5494ca8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.execute;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.sql.ParameterMetaData;
import java.sql.SQLException;
import java.util.Collections;
@@ -325,11 +324,16 @@ public abstract class BaseQueryPlan implements QueryPlan {
ScanUtil.setTenantId(scan, tenantIdBytes);
String customAnnotations =
LogUtil.customAnnotationsToString(connection);
- ScanUtil.setCustomAnnotations(scan, customAnnotations == null ? null
- : customAnnotations.getBytes(StandardCharsets.UTF_8));
- // Set local index related scan attributes.
- if (table.getIndexType() == IndexType.LOCAL) {
- ScanUtil.setLocalIndex(scan);
+ ScanUtil.setCustomAnnotations(scan,
+ customAnnotations == null ? null :
customAnnotations.getBytes());
+ // Set index related scan attributes.
+ if (table.getType() == PTableType.INDEX) {
+ if (table.getIndexType() == IndexType.LOCAL) {
+ ScanUtil.setLocalIndex(scan);
+ } else if (context.isUncoveredIndex()) {
+ ScanUtil.setUncoveredGlobalIndex(scan);
+ }
+
Set<PColumn> dataColumns = context.getDataColumns();
// If any data columns to join back from data table are present
then we set following attributes
// 1. data columns to be projected and their key value schema.
@@ -353,11 +357,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
KeyValueSchema schema =
ProjectedColumnExpression.buildSchema(dataColumns);
// Set key value schema of the data columns.
serializeSchemaIntoScan(scan, schema);
-
- // Set index maintainer of the local index.
- serializeIndexMaintainerIntoScan(scan, dataTable);
- // Set view constants if exists.
- serializeViewConstantsIntoScan(scan, dataTable);
+ if (table.getIndexType() == IndexType.LOCAL) {
+ // Set index maintainer of the local index.
+ serializeIndexMaintainerIntoScan(scan, dataTable);
+ // Set view constants if exists.
+ serializeViewConstantsIntoScan(scan, dataTable);
+ }
}
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 59df237..26fee20 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -281,13 +281,14 @@ public abstract class BaseResultIterators extends
ExplainTable implements Result
}
if (perScanLimit != null) {
- if
(scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER) == null) {
+ if (scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER)
== null) {
ScanUtil.andFilterAtEnd(scan, new
PageFilter(perScanLimit));
} else {
- // if we have a local index filter and a limit, handle the
limit after the filter
+ // if we have an index filter and a limit, handle the
limit after the filter
// we cast the limit to a long even though it passed as an
Integer so that
// if we need extend this in the future the serialization
is unchanged
-
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT,
Bytes.toBytes((long)perScanLimit));
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_LIMIT,
+ Bytes.toBytes((long) perScanLimit));
}
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index c3ab8f1..e0256c9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -191,7 +191,11 @@ public abstract class ExplainTable {
if (whereFilter != null) {
whereFilterStr = whereFilter.toString();
} else {
- byte[] expBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR);
+ byte[] expBytes =
scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR);
+ if (expBytes == null) {
+ // For older clients
+ expBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR);
+ }
if (expBytes != null) {
whereFilterStr = Bytes.toString(expBytes);
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index e258e2a..f371206 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -128,21 +128,15 @@ public class NonAggregateRegionScannerFactory extends
RegionScannerFactory {
PhoenixTransactionContext tx = null;
ColumnReference[] dataColumns =
IndexUtil.deserializeDataTableColumnsToJoin(scan);
if (dataColumns != null) {
- tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
- dataRegion = env.getRegion();
- boolean useProto = false;
- byte[] localIndexBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO);
- useProto = localIndexBytes != null;
- if (localIndexBytes == null) {
- localIndexBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD);
- }
- int clientVersion = ScanUtil.getClientVersion(scan);
- List<IndexMaintainer> indexMaintainers =
- IndexMaintainer.deserialize(localIndexBytes, useProto);
- indexMaintainer = indexMaintainers.get(0);
- viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
- byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
- tx = TransactionFactory.getTransactionContext(txState, clientVersion);
+ tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
+ dataRegion = env.getRegion();
+ int clientVersion = ScanUtil.getClientVersion(scan);
+ List<IndexMaintainer> indexMaintainers =
+ IndexUtil.deSerializeIndexMaintainersFromScan(scan);
+ indexMaintainer = indexMaintainers.get(0);
+ viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
+ tx = TransactionFactory.getTransactionContext(txState, clientVersion);
}
final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
index 3426de8..8e2bde6 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
@@ -135,9 +135,13 @@ public abstract class RegionScannerFactory {
long extraLimit = -1;
{
- // for local indexes construct the row filter for uncovered columns
if it exists
- if (ScanUtil.isLocalIndex(scan)) {
- byte[] expBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER);
+ // for indexes construct the row filter for uncovered columns if it
exists
+ if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan)) {
+ byte[] expBytes =
scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER);
+ if (expBytes == null) {
+ // For older clients
+ expBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER);
+ }
if (expBytes != null) {
try {
ByteArrayInputStream stream = new
ByteArrayInputStream(expBytes);
@@ -149,7 +153,11 @@ public abstract class RegionScannerFactory {
throw new RuntimeException(io);
}
}
- byte[] limitBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT);
+ byte[] limitBytes =
scan.getAttribute(BaseScannerRegionObserver.INDEX_LIMIT);
+ if (limitBytes == null) {
+ // For older clients
+ limitBytes =
scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT);
+ }
if (limitBytes != null) {
extraLimit = Bytes.toLong(limitBytes);
}
@@ -217,7 +225,8 @@ public abstract class RegionScannerFactory {
if (result.size() == 0) {
return next;
}
- if (ScanUtil.isLocalIndex(scan) && !ScanUtil.isAnalyzeTable(scan)) {
+ if ((ScanUtil.isLocalOrUncoveredGlobalIndex(scan))
+ && !ScanUtil.isAnalyzeTable(scan)) {
if(actualStartKey!=null) {
next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs,
result,
null);
@@ -229,8 +238,8 @@ public abstract class RegionScannerFactory {
dataRegion will never be null in case of non-coprocessor call,
therefore no need to refactor
*/
- IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns,
- tupleProjector, dataRegion, indexMaintainer, viewConstants,
ptr);
+ IndexUtil.wrapResultUsingOffset(env, result, scan, offset,
dataColumns,
+ tupleProjector, dataRegion, indexMaintainer, viewConstants,
ptr);
if (extraWhere != null) {
Tuple merged = useQualifierAsListIndex ? new
PositionBasedResultTuple(result) :
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index ad21d9c..bd3282c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -334,6 +334,7 @@ public class QueryOptimizer {
boolean isProjected =
dataPlan.getContext().getResolver().getTables().get(0).getTable().getType() ==
PTableType.PROJECTED;
// Check index state of now potentially updated index table to make
sure it's active
TableRef indexTableRef = resolver.getTables().get(0);
+ indexTableRef.setHinted(isHinted);
PTable indexTable = indexTableRef.getTable();
PIndexState indexState = indexTable.getIndexState();
Map<TableRef, QueryPlan> dataPlans =
Collections.singletonMap(indexTableRef, dataPlan);
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
similarity index 65%
rename from
phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java
rename to
phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
index 87f0999..3f45ebb 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
@@ -23,17 +23,23 @@ import java.util.Set;
import org.apache.phoenix.compile.FromCompiler;
import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.expression.ColumnExpression;
+import org.apache.phoenix.expression.IsNullExpression;
import org.apache.phoenix.expression.ProjectedColumnExpression;
import org.apache.phoenix.parse.ParseNodeFactory;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.util.IndexUtil;
-public class LocalIndexDataColumnRef extends ColumnRef {
+/**
+ * Even when a column is not covered by an index table for a given query, we
may still want to
+ * use index in the query plan and fetch the missing columns from the data
table rows on the
+ * server side. This class is used to keep track of such data columns.
+ */
+public class IndexDataColumnRef extends ColumnRef {
final private int position;
final private Set<PColumn> columns;
private static final ParseNodeFactory FACTORY = new ParseNodeFactory();
- public LocalIndexDataColumnRef(StatementContext context, TableRef tRef,
String indexColumnName)
+ public IndexDataColumnRef(StatementContext context, TableRef tRef, String
indexColumnName)
throws MetaDataEntityNotFoundException, SQLException {
super(FromCompiler.getResolver(
FACTORY.namedTable(
@@ -48,15 +54,15 @@ public class LocalIndexDataColumnRef extends ColumnRef {
columns = context.getDataColumns();
}
- protected LocalIndexDataColumnRef(LocalIndexDataColumnRef
localIndexDataColumnRef, long timestamp) {
- super(localIndexDataColumnRef, timestamp);
- this.position = localIndexDataColumnRef.position;
- this.columns = localIndexDataColumnRef.columns;
+ protected IndexDataColumnRef(IndexDataColumnRef indexDataColumnRef, long
timestamp) {
+ super(indexDataColumnRef, timestamp);
+ this.position = indexDataColumnRef.position;
+ this.columns = indexDataColumnRef.columns;
}
@Override
public ColumnRef cloneAtTimestamp(long timestamp) {
- return new LocalIndexDataColumnRef(this, timestamp);
+ return new IndexDataColumnRef(this, timestamp);
}
@Override
@@ -64,4 +70,25 @@ public class LocalIndexDataColumnRef extends ColumnRef {
String displayName = this.getTableRef().getColumnDisplayName(this,
schemaNameCaseSensitive, colNameCaseSensitive);
return new ProjectedColumnExpression(this.getColumn(), columns,
position, displayName);
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + position;
+ result = prime * result + ((columns == null) ? 0 : columns.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o)) {
+ return false;
+ }
+ IndexDataColumnRef that = (IndexDataColumnRef) o;
+ if (position != that.position) {
+ return false;
+ }
+ return columns.equals(that.columns);
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 5f426b0..1df722f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -38,6 +38,7 @@ public class TableRef {
private final long lowerBoundTimeStamp;
private final boolean hasDynamicCols;
private final long currentTime;
+ private boolean hinted;
private static TableRef createEmptyTableRef() {
try {
@@ -53,15 +54,18 @@ public class TableRef {
}
public TableRef(TableRef tableRef) {
- this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp,
tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp,
+ tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols,
tableRef.hinted);
}
public TableRef(TableRef tableRef, long timeStamp) {
- this(tableRef.alias, tableRef.table, timeStamp,
tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(tableRef.alias, tableRef.table, timeStamp,
tableRef.lowerBoundTimeStamp,
+ tableRef.hasDynamicCols, tableRef.hinted);
}
public TableRef(TableRef tableRef, String alias) {
- this(alias, tableRef.table, tableRef.upperBoundTimeStamp,
tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(alias, tableRef.table, tableRef.upperBoundTimeStamp,
tableRef.lowerBoundTimeStamp,
+ tableRef.hasDynamicCols, tableRef.hinted);
}
public TableRef(PTable table) {
@@ -69,15 +73,20 @@ public class TableRef {
}
public TableRef(PTable table, long upperBoundTimeStamp, long
lowerBoundTimeStamp) {
- this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false);
+ this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false,
false);
}
public TableRef(String alias, PTable table, long upperBoundTimeStamp,
boolean hasDynamicCols) {
- this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols);
+ this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols, false);
+ }
+
+ public TableRef(String alias, PTable table, long upperBoundTimeStamp, long
lowerBoundTimeStamp,
+ boolean hasDynamicCols) {
+ this(alias, table, upperBoundTimeStamp, lowerBoundTimeStamp,
hasDynamicCols, false);
}
- public TableRef(String alias, PTable table, long upperBoundTimeStamp, long
lowerBoundTimeStamp,
- boolean hasDynamicCols) {
+ public TableRef(String alias, PTable table, long upperBoundTimeStamp, long
lowerBoundTimeStamp,
+ boolean hasDynamicCols, boolean hinted) {
this.alias = alias;
this.table = table;
// if UPDATE_CACHE_FREQUENCY is set, always let the server set
timestamps
@@ -85,6 +94,7 @@ public class TableRef {
this.currentTime = this.upperBoundTimeStamp;
this.lowerBoundTimeStamp = lowerBoundTimeStamp;
this.hasDynamicCols = hasDynamicCols;
+ this.hinted = hinted;
}
public PTable getTable() {
@@ -103,6 +113,14 @@ public class TableRef {
return alias;
}
+ public boolean isHinted() {
+ return hinted;
+ }
+
+ public void setHinted(boolean hinted) {
+ this.hinted = hinted;
+ }
+
public String getColumnDisplayName(ColumnRef ref, boolean cfCaseSensitive,
boolean cqCaseSensitive) {
String cf = null;
String cq = null;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 0bab683..c172f68 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -17,6 +17,9 @@
*/
package org.apache.phoenix.util;
+import static
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD;
+import static
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO;
+import static
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME;
import static
org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERSION;
import static
org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION;
import static
org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_PATCH_NUMBER;
@@ -46,6 +49,8 @@ import org.apache.hadoop.hbase.PhoenixTagType;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.phoenix.hbase.index.table.HTableFactory;
+import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.thirdparty.com.google.common.cache.Cache;
import org.apache.phoenix.thirdparty.com.google.common.cache.CacheBuilder;
@@ -459,6 +464,25 @@ public class IndexUtil {
}
}
+ public static List<IndexMaintainer>
deSerializeIndexMaintainersFromScan(Scan scan) {
+ boolean useProto = false;
+ byte[] indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD_PROTO);
+ useProto = indexBytes != null;
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
+ }
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
+ useProto = indexBytes != null;
+ }
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_MD);
+ }
+ List<IndexMaintainer> indexMaintainers =
+ indexBytes == null ? null :
IndexMaintainer.deserialize(indexBytes, useProto);
+ return indexMaintainers;
+ }
+
public static byte[][] deserializeViewConstantsFromScan(Scan scan) {
byte[] bytes =
scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS);
if (bytes == null) return null;
@@ -556,7 +580,7 @@ public class IndexUtil {
}
public static void wrapResultUsingOffset(final
RegionCoprocessorEnvironment environment,
- List<Cell> result, final int offset, ColumnReference[] dataColumns,
+ List<Cell> result, final Scan scan, final int offset,
ColumnReference[] dataColumns,
TupleProjector tupleProjector, Region dataRegion, IndexMaintainer
indexMaintainer,
byte[][] viewConstants, ImmutableBytesWritable ptr) throws
IOException {
if (tupleProjector != null) {
@@ -576,18 +600,27 @@ public class IndexUtil {
}
}
Result joinResult = null;
- if (dataRegion != null) {
- joinResult = dataRegion.get(get);
- } else {
- TableName dataTable =
-
TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
-
environment.getRegion().getTableDesc().getNameAsString()));
- HTableInterface table = null;
- try {
- table = environment.getTable(dataTable);
+ if (ScanUtil.isLocalIndex(scan)) {
+ if (dataRegion != null) {
+ joinResult = dataRegion.get(get);
+ } else {
+ TableName dataTable =
+
TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
+
environment.getRegion().getTableDesc().getNameAsString()));
+ try (Table table = environment.getTable(dataTable)) {
+ joinResult = table.get(get);
+ }
+ }
+ } else if (ScanUtil.isUncoveredGlobalIndex(scan)) {
+ byte[] dataTableName =
scan.getAttribute(PHYSICAL_DATA_TABLE_NAME);
+
+ HTableFactory hTableFactory =
ServerUtil.getDelegateHTableFactory(environment,
+ ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION);
+ try (Table table = hTableFactory.
+ getTable(new ImmutableBytesPtr(dataTableName))) {
joinResult = table.get(get);
} finally {
- if (table != null) table.close();
+ hTableFactory.shutdown();
}
}
// at this point join result has data from the data table. We now
need to take this result and
@@ -981,6 +1014,13 @@ public class IndexUtil {
}
}
+ public static boolean isHintedGlobalIndex(final TableRef tableRef) {
+ PTable table = tableRef.getTable();
+ return table.getType() == PTableType.INDEX
+ && table.getIndexType() == PTable.IndexType.GLOBAL
+ && tableRef.isHinted();
+ }
+
/**
* Updates the EMPTY cell value to VERIFIED for global index table rows.
*/
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index c1d3468..8901c50 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -138,9 +138,20 @@ public class ScanUtil {
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX,
PDataType.TRUE_BYTES);
}
+ public static void setUncoveredGlobalIndex(Scan scan) {
+ scan.setAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX,
PDataType.TRUE_BYTES);
+ }
+
public static boolean isLocalIndex(Scan scan) {
return scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX) !=
null;
}
+ public static boolean isUncoveredGlobalIndex(Scan scan) {
+ return
scan.getAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX) != null;
+ }
+
+ public static boolean isLocalOrUncoveredGlobalIndex(Scan scan) {
+ return isLocalIndex(scan) || isUncoveredGlobalIndex(scan);
+ }
public static boolean isNonAggregateScan(Scan scan) {
return
scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null;
