This is an automated email from the ASF dual-hosted git repository. tdsilva pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new 12c538c PHOENIX-5403 Optimize metadata cache lookup of global tables using a tenant specific connection 12c538c is described below commit 12c538ce2db4779b91bc294ebe8e4880673ff8c2 Author: Thomas D'Silva <tdsi...@apache.org> AuthorDate: Mon Jul 22 22:45:04 2019 -0700 PHOENIX-5403 Optimize metadata cache lookup of global tables using a tenant specific connection --- .../java/org/apache/phoenix/end2end/Array2IT.java | 17 ++--- .../org/apache/phoenix/compile/FromCompiler.java | 6 +- .../phoenix/query/ConnectionQueryServicesImpl.java | 7 +- .../org/apache/phoenix/schema/MetaDataClient.java | 74 ++++++++-------------- 4 files changed, 39 insertions(+), 65 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Array2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Array2IT.java index 52bfb86..0cb60c2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Array2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Array2IT.java @@ -37,6 +37,7 @@ import org.apache.phoenix.schema.types.PhoenixArray; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.StringUtil; +import org.junit.Ignore; import org.junit.Test; public class Array2IT extends ArrayIT { @@ -669,27 +670,19 @@ public class Array2IT extends ArrayIT { } - @Test + @Test // see PHOENIX-5416 + @Ignore public void testArrayRefToLiteral() throws Exception { - Connection conn; - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - conn = DriverManager.getConnection(getUrl(), props); - try { - PreparedStatement stmt = conn.prepareStatement("select ?[2] from \"SYSTEM\".\"catalog\" limit 1"); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + PreparedStatement stmt = conn.prepareStatement("select ?[2] from \"SYSTEM\".\"CATALOG\" limit 1"); Array array = conn.createArrayOf("CHAR", new String[] {"a","b","c"}); stmt.setArray(1, array); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); assertFalse(rs.next()); - } catch (SQLException e) { - } finally { - if (conn != null) { - conn.close(); - } } - } @Test diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index ce0c3a1..a1ee0bf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -174,11 +174,7 @@ public class FromCompiler { NamedTableNode tableNode = NamedTableNode.create(null, baseTable, Collections.<ColumnDef>emptyList()); // Always use non-tenant-specific connection here try { - // We need to always get the latest meta data for the parent table of a create view call to ensure that - // that we're copying the current table meta data as of when the view is created. Once we no longer - // copy the parent meta data, but store only the local diffs (PHOENIX-3534), we will no longer need - // to do this. - SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, tableNode, true, true); + SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, tableNode, true); return visitor; } catch (TableNotFoundException e) { // Used for mapped VIEW, since we won't be able to resolve that. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index fc761dd..4112984 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -228,6 +228,7 @@ import org.apache.phoenix.schema.PSynchronizedMetaData; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.PTableRef; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.ReadOnlyTableException; import org.apache.phoenix.schema.SaltingUtil; @@ -271,6 +272,7 @@ import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.ServerUtil; +import org.apache.phoenix.util.TimeKeeper; import org.apache.phoenix.util.UpgradeUtil; import org.apache.twill.zookeeper.ZKClientService; import org.slf4j.Logger; @@ -642,8 +644,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement throwConnectionClosedIfNullMetaData(); // If existing table isn't older than new table, don't replace // If a client opens a connection at an earlier timestamp, this can happen - PTable existingTable = latestMetaData.getTableRef(new PTableKey(table.getTenantId(), table.getName().getString())).getTable(); + PTableRef existingTableRef = latestMetaData.getTableRef(new PTableKey( + table.getTenantId(), table.getName().getString())); + PTable existingTable = existingTableRef.getTable(); if (existingTable.getTimeStamp() > table.getTimeStamp()) { + existingTableRef.setLastAccessTime(TimeKeeper.SYSTEM.getCurrentTime()); return; } } catch (TableNotFoundException e) {} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 9c4ee2c..4afb0a0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -408,8 +408,6 @@ public class MetaDataClient { INDEX_DISABLE_TIMESTAMP +","+ ASYNC_REBUILD_TIMESTAMP + " " + PLong.INSTANCE.getSqlTypeName() + ") VALUES (?, ?, ?, ?, ?, ?)"; - //TODO: merge INSERT_COLUMN_CREATE_TABLE and INSERT_COLUMN_ALTER_TABLE column when - // the new major release is out. private static final String INSERT_COLUMN_CREATE_TABLE = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + @@ -434,29 +432,6 @@ public class MetaDataClient { IS_ROW_TIMESTAMP + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - private static final String INSERT_COLUMN_ALTER_TABLE = - "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + - TENANT_ID + "," + - TABLE_SCHEM + "," + - TABLE_NAME + "," + - COLUMN_NAME + "," + - COLUMN_FAMILY + "," + - DATA_TYPE + "," + - NULLABLE + "," + - COLUMN_SIZE + "," + - DECIMAL_DIGITS + "," + - ORDINAL_POSITION + "," + - SORT_ORDER + "," + - DATA_TABLE_NAME + "," + // write this both in the column and table rows for access by metadata APIs - ARRAY_SIZE + "," + - VIEW_CONSTANT + "," + - IS_VIEW_REFERENCED + "," + - PK_NAME + "," + // write this both in the column and table rows for access by metadata APIs - KEY_SEQ + "," + - COLUMN_DEF + "," + - COLUMN_QUALIFIER + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - /* * Custom sql to add a column to SYSTEM.CATALOG table during upgrade. * We can't use the regular INSERT_COLUMN_ALTER_TABLE sql because the COLUMN_QUALIFIER column @@ -597,12 +572,23 @@ public class MetaDataClient { String fullTableName = SchemaUtil.getTableName(schemaName, tableName); long tableTimestamp = HConstants.LATEST_TIMESTAMP; long tableResolvedTimestamp = HConstants.LATEST_TIMESTAMP; - try { - tableRef = connection.getTableRef(new PTableKey(tenantId, fullTableName)); - table = tableRef.getTable(); - tableTimestamp = table.getTimeStamp(); - tableResolvedTimestamp = tableRef.getResolvedTimeStamp(); - } catch (TableNotFoundException e) { + int tryCount = 0; + // for tenant specific connection, look up the global table without using tenantId + int maxTryCount = tenantId == null ? 1 : 2; + do { + try { + tableRef = connection.getTableRef(new PTableKey(tenantId, fullTableName)); + table = tableRef.getTable(); + tableTimestamp = table.getTimeStamp(); + tableResolvedTimestamp = tableRef.getResolvedTimeStamp(); + break; + } catch (TableNotFoundException e) { + tenantId = null; + } + } while (++tryCount < maxTryCount); + // reset the tenantId if the global table isn't found in the cache + if (table==null) { + tenantId = systemTable ? null : origTenantId; } // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional @@ -622,10 +608,7 @@ public class MetaDataClient { return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table); } - int maxTryCount = tenantId == null ? 1 : 2; - int tryCount = 0; MetaDataMutationResult result; - // if we are looking up an index on a child view that is inherited from its // parent, then we need to resolve the parent of the child view which will also // load any of its indexes instead of trying to load the inherited view index @@ -655,6 +638,7 @@ public class MetaDataClient { } } else { + tryCount = 0; do { final byte[] schemaBytes = PVarchar.INSTANCE.toBytes(schemaName); final byte[] tableBytes = PVarchar.INSTANCE.toBytes(tableName); @@ -681,6 +665,9 @@ public class MetaDataClient { && result.getTable() == null) { result.setTable(table); } + if (result.getTable()!=null) { + addTableToCache(result); + } return result; } MutationCode code = result.getMutationCode(); @@ -914,16 +901,12 @@ public class MetaDataClient { } else { colUpsert.setString(18, column.getExpressionStr()); } - if (colUpsert.getParameterMetaData().getParameterCount() > 18) { - if (column.getColumnQualifierBytes() == null) { - colUpsert.setNull(19, Types.VARBINARY); - } else { - colUpsert.setBytes(19, column.getColumnQualifierBytes()); - } - } - if (colUpsert.getParameterMetaData().getParameterCount() > 19) { - colUpsert.setBoolean(20, column.isRowTimestamp()); + if (column.getColumnQualifierBytes() == null) { + colUpsert.setNull(19, Types.VARBINARY); + } else { + colUpsert.setBytes(19, column.getColumnQualifierBytes()); } + colUpsert.setBoolean(20, column.isRowTimestamp()); colUpsert.execute(); } @@ -3587,10 +3570,7 @@ public class MetaDataClient { Map<String, Integer> changedCqCounters = new HashMap<>(numCols); if (numCols > 0 ) { StatementContext context = new StatementContext(new PhoenixStatement(connection), resolver); - String addColumnSqlToUse = connection.isRunningUpgrade() - && tableName.equals(PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE) - && schemaName.equals(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA) ? ALTER_SYSCATALOG_TABLE_UPGRADE - : INSERT_COLUMN_ALTER_TABLE; + String addColumnSqlToUse = INSERT_COLUMN_CREATE_TABLE; try (PreparedStatement colUpsert = connection.prepareStatement(addColumnSqlToUse)) { short nextKeySeq = SchemaUtil.getMaxKeySeq(table); for( ColumnDef colDef : columnDefs) {