Repository: phoenix Updated Branches: refs/heads/calcite 9252f64d6 -> a37fafdf9
PHOENIX-2416 Implement multi-tenant tables in Phoenix/Calcite integration Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a37fafdf Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a37fafdf Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a37fafdf Branch: refs/heads/calcite Commit: a37fafdf91c5f383b048ee71309ff79a8d9c0174 Parents: 9252f64 Author: maryannxue <[email protected]> Authored: Sun Nov 22 21:25:28 2015 -0500 Committer: maryannxue <[email protected]> Committed: Sun Nov 22 21:25:28 2015 -0500 ---------------------------------------------------------------------- .../org/apache/phoenix/calcite/CalciteIT.java | 104 +++++++++++++++++-- .../apache/phoenix/calcite/PhoenixSchema.java | 41 +++++--- .../phoenix/calcite/rel/PhoenixTableScan.java | 7 +- .../org/apache/phoenix/compile/ScanRanges.java | 48 +++++++++ 4 files changed, 174 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37fafdf/phoenix-core/src/it/java/org/apache/phoenix/calcite/CalciteIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/calcite/CalciteIT.java b/phoenix-core/src/it/java/org/apache/phoenix/calcite/CalciteIT.java index 6ba0bd6..feab154 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/calcite/CalciteIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/calcite/CalciteIT.java @@ -385,7 +385,11 @@ public class CalciteIT extends BaseClientManagedTimeIT { } protected static final String MULTI_TENANT_TABLE = "multitenant_test_table"; - protected static final String MULTI_TENANT_VIEW = "multitenant_test_view"; + protected static final String MULTI_TENANT_TABLE_INDEX = "idx_multitenant_test_table"; + protected static final String MULTI_TENANT_VIEW1 = "multitenant_test_view1"; + protected static final String MULTI_TENANT_VIEW1_INDEX = "idx_multitenant_test_view1"; + protected static final String MULTI_TENANT_VIEW2 = "multitenant_test_view2"; + protected static final String MULTI_TENANT_VIEW2_INDEX = "idx_multitenant_test_view2"; protected void initMultiTenantTables() throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -411,14 +415,41 @@ public class CalciteIT extends BaseClientManagedTimeIT { stmt.setInt(3, 5); stmt.setInt(4, 6); stmt.execute(); + stmt.setString(1, "20"); + stmt.setString(2, "5"); + stmt.setInt(3, 6); + stmt.setInt(4, 7); + stmt.execute(); + conn.commit(); + + conn.createStatement().execute( + "CREATE INDEX " + MULTI_TENANT_TABLE_INDEX + + " ON " + MULTI_TENANT_TABLE + "(col0) INCLUDE (col1)"); conn.commit(); conn.close(); props.setProperty("TenantId", "10"); conn = DriverManager.getConnection(getUrl(), props); - conn.createStatement().execute("CREATE VIEW " + MULTI_TENANT_VIEW + conn.createStatement().execute("CREATE VIEW " + MULTI_TENANT_VIEW1 + " AS select * from " + MULTI_TENANT_TABLE); conn.commit(); + + conn.createStatement().execute( + "CREATE INDEX " + MULTI_TENANT_VIEW1_INDEX + + " ON " + MULTI_TENANT_VIEW1 + "(col0)"); + conn.commit(); + + conn.close(); + props.setProperty("TenantId", "20"); + conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute("CREATE VIEW " + MULTI_TENANT_VIEW2 + + " AS select * from " + MULTI_TENANT_TABLE + " where col1 > 6"); + conn.commit(); + + conn.createStatement().execute( + "CREATE INDEX " + MULTI_TENANT_VIEW2_INDEX + + " ON " + MULTI_TENANT_VIEW2 + "(col0)"); + conn.commit(); } catch (TableAlreadyExistsException e) { } finally { conn.close(); @@ -1554,18 +1585,28 @@ public class CalciteIT extends BaseClientManagedTimeIT { } @Test public void testMultiTenant() throws Exception { - Properties props = getConnectionProps(false); + Properties props = getConnectionProps(true); start(props).sql("select * from " + MULTI_TENANT_TABLE) .explainIs("PhoenixToEnumerableConverter\n" + " PhoenixTableScan(table=[[phoenix, MULTITENANT_TEST_TABLE]])\n") .resultIs(new Object[][] { {"10", "2", 3, 4}, {"15", "3", 4, 5}, - {"20", "4", 5, 6}}) + {"20", "4", 5, 6}, + {"20", "5", 6, 7}}) + .close(); + + start(props).sql("select * from " + MULTI_TENANT_TABLE + " where tenant_id = '20' and col0 > 1") + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixServerProject(TENANT_ID=[$0], ID=[$2], COL0=[CAST($1):INTEGER], COL1=[$3])\n" + + " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_TABLE]], filter=[AND(=(CAST($0):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\" NOT NULL, '20'), >(CAST($1):INTEGER, 1))])\n") + .resultIs(new Object[][] { + {"20", "4", 5, 6}, + {"20", "5", 6, 7}}) .close(); try { - start(props).sql("select * from " + MULTI_TENANT_VIEW) + start(props).sql("select * from " + MULTI_TENANT_VIEW1) .explainIs("") .close(); fail("Should have got SQLException."); @@ -1580,8 +1621,16 @@ public class CalciteIT extends BaseClientManagedTimeIT { {"3", 4, 5}}) .close(); + start(props).sql("select * from " + MULTI_TENANT_TABLE + " where col0 > 1") + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixServerProject(ID=[$1], COL0=[CAST($0):INTEGER], COL1=[$2])\n" + + " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_TABLE]], filter=[>(CAST($0):INTEGER, 1)])\n") + .resultIs(new Object[][] { + {"3", 4, 5}}) + .close(); + try { - start(props).sql("select * from " + MULTI_TENANT_VIEW) + start(props).sql("select * from " + MULTI_TENANT_VIEW1) .explainIs("") .close(); fail("Should have got SQLException."); @@ -1589,12 +1638,53 @@ public class CalciteIT extends BaseClientManagedTimeIT { } props.setProperty("TenantId", "10"); - start(props).sql("select * from " + MULTI_TENANT_VIEW) + start(props).sql("select * from " + MULTI_TENANT_VIEW1) .explainIs("PhoenixToEnumerableConverter\n" + " PhoenixTableScan(table=[[phoenix, MULTITENANT_TEST_TABLE]])\n") .resultIs(new Object[][] { {"2", 3, 4}}) .close(); + + start(props).sql("select * from " + MULTI_TENANT_TABLE + " where col0 > 1") + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixServerProject(ID=[$1], COL0=[CAST($0):INTEGER], COL1=[$2])\n" + + " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_TABLE]], filter=[>(CAST($0):INTEGER, 1)])\n") + .resultIs(new Object[][] { + {"2", 3, 4}}) + .close(); + + start(props).sql("select id, col0 from " + MULTI_TENANT_TABLE + " where col0 > 1") + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixServerProject(ID=[$1], COL0=[CAST($0):INTEGER])\n" + + " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_VIEW1]], filter=[>(CAST($0):INTEGER, 1)])\n") + .resultIs(new Object[][] { + {"2", 3}}) + .close(); + + start(props).sql("select id, col0 from " + MULTI_TENANT_VIEW1 + " where col0 > 1") + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixServerProject(ID=[$1], COL0=[CAST($0):INTEGER])\n" + + " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_VIEW1]], filter=[>(CAST($0):INTEGER, 1)])\n") + .resultIs(new Object[][] { + {"2", 3}}) + .close(); + + props.setProperty("TenantId", "20"); + start(props).sql("select * from " + MULTI_TENANT_VIEW2) + .explainIs("PhoenixToEnumerableConverter\n" + + " PhoenixTableScan(table=[[phoenix, MULTITENANT_TEST_TABLE]], filter=[>($2, 6)])\n") + .resultIs(new Object[][] { + {"5", 6, 7}}) + .close(); + + // TODO disable this test case for now. FilterToProjectUnifyRule might be buggy. + //start(props).sql("select id, col0 from " + MULTI_TENANT_VIEW2 + " where col0 > 1") + // .explainIs("PhoenixToEnumerableConverter\n" + + // " PhoenixServerProject(ID=[$1], COL0=[CAST($0):INTEGER])\n" + + // " PhoenixTableScan(table=[[phoenix, IDX_MULTITENANT_TEST_VIEW2]], filter=[>(CAST($0):INTEGER, 1)])\n") + // .resultIs(new Object[][] { + // {"5", 6}}) + // .close(); } /** Tests a simple command that is defined in Phoenix's extended SQL parser. http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37fafdf/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java index ef14b45..0c45a25 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java @@ -20,9 +20,11 @@ import org.apache.phoenix.parse.TableName; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTable.ViewType; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.util.IndexUtil; @@ -102,18 +104,22 @@ public class PhoenixSchema implements Schema { String viewType = rs.getString(PhoenixDatabaseMetaData.VIEW_TYPE); if (!tableType.equals(PTableType.VIEW.getValue().getString()) || ViewType.MAPPED.name().equals(viewType)) { - ColumnResolver x = FromCompiler.getResolver( - NamedTableNode.create( - null, - TableName.create(schemaName, tableName), - ImmutableList.<ColumnDef>of()), pc); - final List<TableRef> tables = x.getTables(); - assert tables.size() == 1; - PTable pTable = tables.get(0).getTable(); - if (pc.getTenantId() == null && pTable.isMultiTenant()) { - pTable = fixTableMultiTenancy(pTable); + try { + ColumnResolver x = FromCompiler.getResolver( + NamedTableNode.create( + null, + TableName.create(schemaName, tableName), + ImmutableList.<ColumnDef>of()), pc); + final List<TableRef> tables = x.getTables(); + assert tables.size() == 1; + PTable pTable = tables.get(0).getTable(); + if (pc.getTenantId() == null && pTable.isMultiTenant()) { + pTable = fixTableMultiTenancy(pTable); + } + tableMap.put(tableName, pTable); + } catch (TableNotFoundException e) { + // Multi-tenant table with non-tenant-specific connection. } - tableMap.put(tableName, pTable); } else { boolean isMultiTenant = rs.getBoolean(PhoenixDatabaseMetaData.MULTI_TENANT); if (pc.getTenantId() != null || !isMultiTenant) { @@ -125,7 +131,10 @@ public class PhoenixSchema implements Schema { + (schemaName == null ? " is null" : " = '" + schemaName + "'") + " and " + PhoenixDatabaseMetaData.TABLE_NAME + " = '" + tableName + "'" - + " and " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " is not null"; + + " and " + PhoenixDatabaseMetaData.COLUMN_FAMILY + + " is not null" + + " and " + PhoenixDatabaseMetaData.LINK_TYPE + + " = " + LinkType.PHYSICAL_TABLE.getSerializedValue(); ResultSet rs2 = pc.createStatement().executeQuery(q); if (!rs2.next()) { throw new SQLException("View link not found for " + tableName); @@ -234,13 +243,13 @@ public class PhoenixSchema implements Schema { public void defineIndexesAsMaterializations(CalciteSchema calciteSchema) { List<String> path = calciteSchema.path(null); for (PTable table : tableMap.values()) { - for (PTable index : table.getIndexes()) { - addMaterialization(table, index, path, calciteSchema); + if (table.getType() == PTableType.INDEX) { + addMaterialization(table, path, calciteSchema); } } } - protected void addMaterialization(PTable table, PTable index, List<String> path, + protected void addMaterialization(PTable index, List<String> path, CalciteSchema calciteSchema) { StringBuffer sb = new StringBuffer(); sb.append("SELECT"); @@ -251,7 +260,7 @@ public class PhoenixSchema implements Schema { sb.append(" ").append("\"").append(indexColumnName).append("\""); } sb.setCharAt(6, ' '); // replace first comma with space. - sb.append(" FROM ").append("\"").append(table.getTableName().getString()).append("\""); + sb.append(" FROM ").append("\"").append(index.getParentName().getString()).append("\""); MaterializationService.instance().defineMaterialization( calciteSchema, null, sb.toString(), path, index.getTableName().getString(), true, true); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37fafdf/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java index fcf15f9..ec62221 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java @@ -177,7 +177,8 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { filteredRowCount = 1.0; } else if (scanRanges.getBoundPkColumnCount() > 0) { // TODO - filteredRowCount = rowCount * RelMetadataQuery.getSelectivity(this, filter); + int pkCount = scanRanges.getBoundPkColumnCount(); + filteredRowCount = rowCount * Math.pow(RelMetadataQuery.getSelectivity(this, filter), pkCount); } } if (filteredRowCount != null) { @@ -241,8 +242,8 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { columnRefList = ImmutableIntList.copyOf(bitSet.asList()); filterExpr = CalciteUtils.toExpression(filter, implementor); } - filterExpr = WhereOptimizer.pushKeyExpressionsToScan(context, select, filterExpr); - WhereCompiler.setScanFilter(context, select, filterExpr, true, false); + Expression rem = WhereOptimizer.pushKeyExpressionsToScan(context, select, filterExpr); + WhereCompiler.setScanFilter(context, select, rem, true, false); // TODO This is not absolutely strict. We may have a filter like: // pk = '0' and pk = $cor0 where $cor0 happens to get a sample value // as '0', thus making the below test return false and adding an http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37fafdf/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 2a032a3..fa95d88 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -40,6 +40,7 @@ import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.ScanUtil.BytesComparator; import org.apache.phoenix.util.SchemaUtil; +import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -573,6 +574,53 @@ public class ScanRanges { return count; } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ScanRanges)) + return false; + if (this == o) + return true; + + ScanRanges that = (ScanRanges) o; + return Objects.equal(this.filter, that.filter) + && Objects.equal(this.ranges, that.ranges) + && Arrays.equals(this.slotSpan, that.slotSpan) + && Objects.equal(this.schema, that.schema) + && this.isPointLookup == that.isPointLookup + && this.isSalted == that.isSalted + && this.useSkipScanFilter == that.useSkipScanFilter + && Objects.equal(this.scanRange, that.scanRange) + && Objects.equal(this.minMaxRange, that.minMaxRange); + } + + @Override + public int hashCode() { + int prime = 31; + int result = (this.isPointLookup ? 1 : 0) + + ((this.isSalted ? 1 : 0) * 2) + + ((this.useSkipScanFilter ? 1 : 0) * 4); + if (this.filter != null) { + result = result * prime + this.filter.hashCode(); + } + if (this.ranges != null) { + result = result * prime + this.ranges.hashCode(); + } + if (this.slotSpan != null) { + result = result * prime + Arrays.hashCode(this.slotSpan); + } + if (this.schema != null) { + result = result * prime + this.schema.hashCode(); + } + if (this.scanRange != null) { + result = result * prime + this.scanRange.hashCode(); + } + if (this.minMaxRange != null) { + result = result * prime + this.minMaxRange.hashCode(); + } + + return result; + } @Override public String toString() {
