PHOENIX-3421 Column name lookups fail when on an indexed table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49914c2e Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49914c2e Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49914c2e Branch: refs/heads/master Commit: 49914c2e8062bf6b175dc987fac97b3f0e659dbe Parents: a6c9024 Author: James Taylor <[email protected]> Authored: Thu Nov 3 16:21:28 2016 -0700 Committer: James Taylor <[email protected]> Committed: Thu Nov 3 16:47:01 2016 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/QueryMoreIT.java | 4 +- .../org/apache/phoenix/util/PhoenixRuntime.java | 132 ++++++++++++++++++- .../phoenix/util/PhoenixEncodeDecodeTest.java | 4 +- .../apache/phoenix/util/PhoenixRuntimeTest.java | 8 +- 4 files changed, 137 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java index b9162de..2b27f00 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java @@ -275,7 +275,7 @@ public class QueryMoreIT extends ParallelStatsDisabledIT { values[i] = rs.getObject(i + 1); } conn = getTenantSpecificConnection(tenantId); - pkIds.add(Base64.encodeBytes(PhoenixRuntime.encodeValues(conn, tableOrViewName.toUpperCase(), values, columns))); + pkIds.add(Base64.encodeBytes(PhoenixRuntime.encodeColumnValues(conn, tableOrViewName.toUpperCase(), values, columns))); } return pkIds.toArray(new String[pkIds.size()]); } @@ -293,7 +293,7 @@ public class QueryMoreIT extends ParallelStatsDisabledIT { PreparedStatement stmt = conn.prepareStatement(query); int bindCounter = 1; for (int i = 0; i < cursorIds.length; i++) { - Object[] pkParts = PhoenixRuntime.decodeValues(conn, tableName.toUpperCase(), Base64.decode(cursorIds[i]), columns); + Object[] pkParts = PhoenixRuntime.decodeColumnValues(conn, tableName.toUpperCase(), Base64.decode(cursorIds[i]), columns); for (int j = 0; j < pkParts.length; j++) { stmt.setObject(bindCounter++, pkParts[j]); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java index 5dd4592..dbac76f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java @@ -816,7 +816,7 @@ public class PhoenixRuntime { public static List<Pair<String, String>> getPkColsForSql(Connection conn, QueryPlan plan) throws SQLException { checkNotNull(plan); checkNotNull(conn); - List<PColumn> pkColumns = getPkColumns(plan.getTableRef().getTable(), conn, true); + List<PColumn> pkColumns = getPkColumns(plan.getTableRef().getTable(), conn); List<Pair<String, String>> columns = Lists.newArrayListWithExpectedSize(pkColumns.size()); String columnName; String familyName; @@ -921,6 +921,7 @@ public class PhoenixRuntime { return sqlTypeName; } + @Deprecated private static List<PColumn> getPkColumns(PTable ptable, Connection conn, boolean forDataTable) throws SQLException { PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class); List<PColumn> pkColumns = ptable.getPKColumns(); @@ -943,6 +944,28 @@ public class PhoenixRuntime { return pkColumns; } + private static List<PColumn> getPkColumns(PTable ptable, Connection conn) throws SQLException { + PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class); + List<PColumn> pkColumns = ptable.getPKColumns(); + + // Skip the salting column and the view index id column if present. + // Skip the tenant id column too if the connection is tenant specific and the table used by the query plan is multi-tenant + int offset = (ptable.getBucketNum() == null ? 0 : 1) + (ptable.isMultiTenant() && pConn.getTenantId() != null ? 1 : 0) + (ptable.getViewIndexId() == null ? 0 : 1); + + // get a sublist of pkColumns by skipping the offset columns. + pkColumns = pkColumns.subList(offset, pkColumns.size()); + + if (ptable.getType() == PTableType.INDEX) { + // index tables have the same schema name as their parent/data tables. + String fullDataTableName = ptable.getParentName().getString(); + + // Get the corresponding columns of the data table. + List<PColumn> dataColumns = IndexUtil.getDataColumns(fullDataTableName, pkColumns, pConn); + pkColumns = dataColumns; + } + return pkColumns; + } + /** * * @param conn connection that was used for reading/generating value. @@ -955,6 +978,7 @@ public class PhoenixRuntime { * @throws SQLException * @see {@link #decodeValues(Connection, String, byte[], List)} */ + @Deprecated public static byte[] encodeValues(Connection conn, String fullTableName, Object[] values, List<Pair<String, String>> columns) throws SQLException { PTable table = getTable(conn, fullTableName); List<PColumn> pColumns = getPColumns(table, columns); @@ -978,7 +1002,7 @@ public class PhoenixRuntime { * * @param conn connection that was used for reading/generating value. * @param fullTableName fully qualified table name - * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeValues(Connection, String, Object[], List)} + * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeColumnValues(Connection, String, Object[], List)} * @param columns list of column names for the columns that have their respective values * present in the byte array. The column names should be in the same order as their values are in the byte array. * The column name includes both family name, if present, and column name. @@ -986,6 +1010,7 @@ public class PhoenixRuntime { * @throws SQLException * */ + @Deprecated public static Object[] decodeValues(Connection conn, String fullTableName, byte[] value, List<Pair<String, String>> columns) throws SQLException { PTable table = getTable(conn, fullTableName); KeyValueSchema kvSchema = buildKeyValueSchema(getPColumns(table, columns)); @@ -1007,6 +1032,70 @@ public class PhoenixRuntime { return values.toArray(); } + /** + * + * @param conn connection that was used for reading/generating value. + * @param fullTableName fully qualified table name + * @param values values of the columns + * @param columns list of pair of column that includes column family as first part and column name as the second part. + * Column family is optional and hence nullable. Columns in the list have to be in the same order as the order of occurence + * of their values in the object array. + * @return values encoded in a byte array + * @throws SQLException + * @see {@link #decodeValues(Connection, String, byte[], List)} + */ + public static byte[] encodeColumnValues(Connection conn, String fullTableName, Object[] values, List<Pair<String, String>> columns) throws SQLException { + PTable table = getTable(conn, fullTableName); + List<PColumn> pColumns = getColumns(table, columns); + List<Expression> expressions = new ArrayList<Expression>(pColumns.size()); + int i = 0; + for (PColumn col : pColumns) { + Object value = values[i]; + // for purposes of encoding, sort order of the columns doesn't matter. + Expression expr = LiteralExpression.newConstant(value, col.getDataType(), col.getMaxLength(), col.getScale()); + expressions.add(expr); + i++; + } + KeyValueSchema kvSchema = buildKeyValueSchema(pColumns); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + ValueBitSet valueSet = ValueBitSet.newInstance(kvSchema); + return kvSchema.toBytes(expressions.toArray(new Expression[0]), valueSet, ptr); + } + + + /** + * + * @param conn connection that was used for reading/generating value. + * @param fullTableName fully qualified table name + * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeColumnValues(Connection, String, Object[], List)} + * @param columns list of column names for the columns that have their respective values + * present in the byte array. The column names should be in the same order as their values are in the byte array. + * The column name includes both family name, if present, and column name. + * @return decoded values for each column + * @throws SQLException + * + */ + public static Object[] decodeColumnValues(Connection conn, String fullTableName, byte[] value, List<Pair<String, String>> columns) throws SQLException { + PTable table = getTable(conn, fullTableName); + KeyValueSchema kvSchema = buildKeyValueSchema(getColumns(table, columns)); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(value); + ValueBitSet valueSet = ValueBitSet.newInstance(kvSchema); + valueSet.clear(); + valueSet.or(ptr); + int maxOffset = ptr.getOffset() + ptr.getLength(); + Boolean hasValue; + kvSchema.iterator(ptr); + int i = 0; + List<Object> values = new ArrayList<Object>(); + while(hasValue = kvSchema.next(ptr, i, maxOffset, valueSet) != null) { + if(hasValue) { + values.add(kvSchema.getField(i).getDataType().toObject(ptr)); + } + i++; + } + return values.toArray(); + } + private static KeyValueSchema buildKeyValueSchema(List<PColumn> columns) { KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(getMinNullableIndex(columns)); for (PColumn col : columns) { @@ -1026,13 +1115,14 @@ public class PhoenixRuntime { return minNullableIndex; } - /** + /** * @param table table to get the {@code PColumn} for * @param columns list of pair of column that includes column family as first part and column name as the second part. * Column family is optional and hence nullable. * @return list of {@code PColumn} for fullyQualifiedColumnNames * @throws SQLException */ + @Deprecated private static List<PColumn> getPColumns(PTable table, List<Pair<String, String>> columns) throws SQLException { List<PColumn> pColumns = new ArrayList<PColumn>(columns.size()); for (Pair<String, String> column : columns) { @@ -1041,6 +1131,7 @@ public class PhoenixRuntime { return pColumns; } + @Deprecated private static PColumn getPColumn(PTable table, @Nullable String familyName, String columnName) throws SQLException { if (table==null) { throw new SQLException("Table must not be null."); @@ -1051,6 +1142,41 @@ public class PhoenixRuntime { // normalize and remove quotes from family and column names before looking up. familyName = SchemaUtil.normalizeIdentifier(familyName); columnName = SchemaUtil.normalizeIdentifier(columnName); + PColumn pColumn = null; + if (familyName != null) { + PColumnFamily family = table.getColumnFamily(familyName); + pColumn = family.getColumn(columnName); + } else { + pColumn = table.getColumn(columnName); + } + return pColumn; + } + + /** + * @param table table to get the {@code PColumn} for + * @param columns list of pair of column that includes column family as first part and column name as the second part. + * Column family is optional and hence nullable. + * @return list of {@code PColumn} for fullyQualifiedColumnNames + * @throws SQLException + */ + private static List<PColumn> getColumns(PTable table, List<Pair<String, String>> columns) throws SQLException { + List<PColumn> pColumns = new ArrayList<PColumn>(columns.size()); + for (Pair<String, String> column : columns) { + pColumns.add(getColumn(table, column.getFirst(), column.getSecond())); + } + return pColumns; + } + + private static PColumn getColumn(PTable table, @Nullable String familyName, String columnName) throws SQLException { + if (table==null) { + throw new SQLException("Table must not be null."); + } + if (columnName==null) { + throw new SQLException("columnName must not be null."); + } + // normalize and remove quotes from family and column names before looking up. + familyName = SchemaUtil.normalizeIdentifier(familyName); + columnName = SchemaUtil.normalizeIdentifier(columnName); // Column names are always for the data table, so we must translate them if // we're dealing with an index table. if (table.getType() == PTableType.INDEX) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java index 85338c4..56b3f45 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java @@ -62,8 +62,8 @@ public class PhoenixEncodeDecodeTest extends BaseConnectionlessQueryTest { Date d = nullFixedWidth ? null : new Date(100); String s = nullVariableWidth ? null : "foo"; Object[] values = new Object[] {"def", "eid", d, s, s}; - byte[] bytes = PhoenixRuntime.encodeValues(conn, "T", values, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1"))); - Object[] decodedValues = PhoenixRuntime.decodeValues(conn, "T", bytes, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1"))); + byte[] bytes = PhoenixRuntime.encodeColumnValues(conn, "T", values, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1"))); + Object[] decodedValues = PhoenixRuntime.decodeColumnValues(conn, "T", bytes, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1"))); assertEquals(Lists.newArrayList("def", "eid", d, s, s), Arrays.asList(decodedValues)); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java index 783ab17..430c20b 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java @@ -131,8 +131,8 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest { List<Pair<String,String>> pkColumns = PhoenixRuntime.getPkColsForSql(conn, plan); String fullTableName = plan.getTableRef().getTable().getName().getString(); assertEquals("I", fullTableName); - byte[] encodedValues = PhoenixRuntime.encodeValues(conn, fullTableName, values, pkColumns); - Object[] decodedValues = PhoenixRuntime.decodeValues(conn, fullTableName, encodedValues, pkColumns); + byte[] encodedValues = PhoenixRuntime.encodeColumnValues(conn, fullTableName, values, pkColumns); + Object[] decodedValues = PhoenixRuntime.decodeColumnValues(conn, fullTableName, encodedValues, pkColumns); assertArrayEquals(values, decodedValues); plan = conn.createStatement().unwrap(PhoenixStatement.class).optimizeQuery("SELECT /*+ NO_INDEX */ ENTITY_HISTORY_ID FROM T"); @@ -140,8 +140,8 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest { values = new Object[] {tenantId, parentId, createdDate, ehId}; fullTableName = plan.getTableRef().getTable().getName().getString(); assertEquals("T", fullTableName); - encodedValues = PhoenixRuntime.encodeValues(conn, fullTableName, values, pkColumns); - decodedValues = PhoenixRuntime.decodeValues(conn, fullTableName, encodedValues, pkColumns); + encodedValues = PhoenixRuntime.encodeColumnValues(conn, fullTableName, values, pkColumns); + decodedValues = PhoenixRuntime.decodeColumnValues(conn, fullTableName, encodedValues, pkColumns); assertArrayEquals(values, decodedValues); }
