This is an automated email from the ASF dual-hosted git repository.
shahrs87 pushed a commit to branch PHOENIX-6883-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-6883-feature by this
push:
new 4f8811b456 PHOENIX-7212 : Handle inherited view indexes when
validating LAST_DDL_TIMESTAMPS (#1823)
4f8811b456 is described below
commit 4f8811b4562fb062f1bc88416e932ea2be9743ec
Author: palash <[email protected]>
AuthorDate: Wed Feb 14 09:50:43 2024 -0800
PHOENIX-7212 : Handle inherited view indexes when validating
LAST_DDL_TIMESTAMPS (#1823)
---
.../org/apache/phoenix/execute/MutationState.java | 2 +-
.../org/apache/phoenix/jdbc/PhoenixStatement.java | 2 +-
.../phoenix/util/ValidateLastDDLTimestampUtil.java | 60 ++++++++-------
.../java/org/apache/phoenix/util/ViewUtil.java | 3 +-
.../phoenix/cache/ServerMetadataCacheTest.java | 90 ++++++++++++++++++++++
.../apache/phoenix/compile/WhereOptimizerTest.java | 43 +++++++++++
6 files changed, 170 insertions(+), 30 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
index 60df3a570f..a4341b91a9 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -1219,7 +1219,7 @@ public class MutationState implements SQLCloseable {
List<TableRef> tableRefs = new
ArrayList<>(this.mutationsMap.keySet());
try {
ValidateLastDDLTimestampUtil.validateLastDDLTimestamp(
- connection, tableRefs, true, true);
+ connection, tableRefs, true);
} catch (StaleMetadataCacheException e) {
GlobalClientMetrics
.GLOBAL_CLIENT_STALE_METADATA_CACHE_EXCEPTION_COUNTER.increment();
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index aec3b4d78d..fa59125e86 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -389,7 +389,7 @@ public class PhoenixStatement implements
PhoenixMonitoredStatement, SQLCloseable
//plan.getTableRef can be null in some cases
like EXPLAIN <query>
if (shouldValidateLastDdlTimestamp &&
plan.getTableRef() != null) {
ValidateLastDDLTimestampUtil.validateLastDDLTimestamp(
- connection,
Arrays.asList(plan.getTableRef()), false, true);
+ connection,
Arrays.asList(plan.getTableRef()), true);
}
if (plan.getTableRef() != null
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
index 01d58f2f11..7d77e6aa89 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
import org.apache.phoenix.exception.StaleMetadataCacheException;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PName;
@@ -74,15 +75,14 @@ public class ValidateLastDDLTimestampUtil {
* Verifies that table metadata for given tables is up-to-date in client
cache with server.
* A random live region server is picked for invoking the RPC to validate
LastDDLTimestamp.
* Retry once if there was an error performing the RPC, otherwise throw
the Exception.
+ *
* @param tableRefs
- * @param isWritePath
* @param doRetry
* @throws SQLException
*/
- public static void validateLastDDLTimestamp(
- PhoenixConnection conn, List<TableRef> tableRefs, boolean
isWritePath, boolean doRetry)
- throws SQLException {
-
+ public static void validateLastDDLTimestamp(PhoenixConnection conn,
+ List<TableRef> tableRefs,
+ boolean doRetry) throws
SQLException {
String infoString = getInfoString(conn.getTenantId(), tableRefs);
try (Admin admin = conn.getQueryServices().getAdmin()) {
// get all live region servers
@@ -100,7 +100,7 @@ public class ValidateLastDDLTimestampUtil {
service =
RegionServerEndpointProtos.RegionServerEndpointService
.newBlockingStub(admin.coprocessorService(regionServer));
RegionServerEndpointProtos.ValidateLastDDLTimestampRequest request
- = getValidateDDLTimestampRequest(conn, tableRefs,
isWritePath);
+ = getValidateDDLTimestampRequest(tableRefs);
service.validateLastDDLTimestamp(null, request);
} catch (Exception e) {
SQLException parsedException = ClientUtil.parseServerException(e);
@@ -112,7 +112,7 @@ public class ValidateLastDDLTimestampUtil {
if (doRetry) {
// update the list of live region servers
conn.getQueryServices().refreshLiveRegionServers();
- validateLastDDLTimestamp(conn, tableRefs, isWritePath, false);
+ validateLastDDLTimestamp(conn, tableRefs, false);
return;
}
throw parsedException;
@@ -122,19 +122,16 @@ public class ValidateLastDDLTimestampUtil {
/**
* Build a request for the validateLastDDLTimestamp RPC for the given
tables.
* 1. For a view, we need to add all its ancestors to the request
- * in case something changed in the hierarchy.
+ * in case something changed in the hierarchy.
* 2. For an index, we need to add its parent table to the request
- * in case the index was dropped.
- * 3. On the write path, we need to add all indexes of a table/view
- * in case index state was changed.
- * @param conn
+ * in case the index was dropped.
+ * 3. Add all indexes of a table/view in case index state was changed.
+ *
* @param tableRefs
- * @param isWritePath
* @return ValidateLastDDLTimestampRequest for the table in tableRef
*/
private static RegionServerEndpointProtos.ValidateLastDDLTimestampRequest
- getValidateDDLTimestampRequest(PhoenixConnection conn, List<TableRef>
tableRefs,
- boolean isWritePath) {
+ getValidateDDLTimestampRequest(List<TableRef> tableRefs) {
RegionServerEndpointProtos.ValidateLastDDLTimestampRequest.Builder
requestBuilder
=
RegionServerEndpointProtos.ValidateLastDDLTimestampRequest.newBuilder();
@@ -158,18 +155,15 @@ public class ValidateLastDDLTimestampUtil {
PTable ptable = tableRef.getTable();
innerBuilder =
RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder();
setLastDDLTimestampRequestParameters(innerBuilder, ptable.getKey(),
-
ptable.getLastDDLTimestamp());
+ ptable.getLastDDLTimestamp());
requestBuilder.addLastDDLTimestampRequests(innerBuilder);
- //on the write path, we need to validate all indexes of a
table/view
- //in case index state was changed
- if (isWritePath) {
- for (PTable idxPTable : tableRef.getTable().getIndexes()) {
- innerBuilder =
RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder();
- setLastDDLTimestampRequestParameters(innerBuilder,
idxPTable.getKey(),
-
idxPTable.getLastDDLTimestamp());
- requestBuilder.addLastDDLTimestampRequests(innerBuilder);
- }
+ // add all indexes of the current table
+ for (PTable idxPTable : tableRef.getTable().getIndexes()) {
+ innerBuilder =
RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder();
+ setLastDDLTimestampRequestParameters(innerBuilder,
idxPTable.getKey(),
+ idxPTable.getLastDDLTimestamp());
+ requestBuilder.addLastDDLTimestampRequests(innerBuilder);
}
}
return requestBuilder.build();
@@ -181,15 +175,27 @@ public class ValidateLastDDLTimestampUtil {
private static void setLastDDLTimestampRequestParameters(
RegionServerEndpointProtos.LastDDLTimestampRequest.Builder builder,
PTableKey key, long lastDDLTimestamp) {
+ String tableName = key.getTableName();
+ String schemaName = key.getSchemaName();
+
+ // view(V) with Index (VIndex) -> child view (V1) -> grand child view
(V2)
+ // inherited view index is of the form V2#V1#VIndex, it does not exist
in syscat
+ if
(tableName.contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR)) {
+ int lastIndexOf =
tableName.lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR);
+ String indexFullName = tableName.substring(lastIndexOf + 1);
+ tableName = SchemaUtil.getTableNameFromFullName(indexFullName);
+ schemaName = SchemaUtil.getSchemaNameFromFullName(indexFullName);
+ }
+
byte[] tenantIDBytes = key.getTenantId() == null
? HConstants.EMPTY_BYTE_ARRAY
: key.getTenantId().getBytes();
- byte[] schemaBytes = key.getSchemaName() == null
+ byte[] schemaBytes = schemaName == null
? HConstants.EMPTY_BYTE_ARRAY
: key.getSchemaName().getBytes();
builder.setTenantId(ByteStringer.wrap(tenantIDBytes));
builder.setSchemaName(ByteStringer.wrap(schemaBytes));
- builder.setTableName(ByteStringer.wrap(key.getTableName().getBytes()));
+ builder.setTableName(ByteStringer.wrap(tableName.getBytes()));
builder.setLastDDLTimestamp(lastDDLTimestamp);
}
}
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
index 505b9f3ad1..aadba4a8e6 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
@@ -484,7 +484,8 @@ public class ViewUtil {
.setTableName(modifiedIndexName)
.setViewStatement(viewStatement)
.setUpdateCacheFrequency(view.getUpdateCacheFrequency())
- .setTenantId(view.getTenantId())
+ //retain the tenantId from the index being
inherited
+ .setTenantId(index.getTenantId())
.setPhysicalNames(Collections.singletonList(index.getPhysicalName()))
.build());
}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java
index ec70879b17..48042655f3 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java
@@ -20,12 +20,15 @@ package org.apache.phoenix.cache;
import org.apache.hadoop.hbase.util.Bytes;
import
org.apache.phoenix.coprocessorclient.metrics.MetricsMetadataCachingSource;
import
org.apache.phoenix.coprocessorclient.metrics.MetricsPhoenixCoprocessorSourceFactory;
+import org.apache.phoenix.end2end.IndexToolIT;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.monitoring.GlobalClientMetrics;
import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ConnectionProperty;
@@ -1591,8 +1594,95 @@ public class ServerMetadataCacheTest extends
ParallelStatsDisabledIT {
}
}
+ /**
+ * Test that tenant connections are able to learn about state change of an
inherited index
+ * on their tenant views with different names.
+ */
+ @Test
+ public void testInheritedIndexOnTenantViewsDifferentNames() throws
Exception {
+ testInheritedIndexOnTenantViews(false);
+ }
+
+ /**
+ * Test that tenant connections are able to learn about state change of an
inherited index
+ * on their tenant views with same names.
+ */
+ @Test
+ public void testInheritedIndexOnTenantViewsSameNames() throws Exception {
+ testInheritedIndexOnTenantViews(true);
+ }
+
+ public void testInheritedIndexOnTenantViews(boolean sameTenantViewNames)
throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String url = QueryUtil.getConnectionUrl(props, config, "client1");
+ ConnectionQueryServices cqs = driver.getConnectionQueryServices(url,
props);
+ String baseTableName = generateUniqueName();
+ String globalViewName = generateUniqueName();
+ String globalViewIndexName = generateUniqueName();
+ String tenantViewName1 = generateUniqueName();
+ String tenantViewName2 = sameTenantViewNames ? tenantViewName1 :
generateUniqueName();
+ try (Connection conn = cqs.connect(url, props)) {
+ // create table, view and view index
+ conn.createStatement().execute("CREATE TABLE " + baseTableName +
+ " (TENANT_ID CHAR(9) NOT NULL, KP CHAR(3) NOT NULL, PK
CHAR(3) NOT NULL, KV CHAR(2), KV2 CHAR(2) " +
+ "CONSTRAINT PK PRIMARY KEY(TENANT_ID, KP, PK))
MULTI_TENANT=true,UPDATE_CACHE_FREQUENCY=NEVER");
+ conn.createStatement().execute("CREATE VIEW " + globalViewName +
+ " AS SELECT * FROM " + baseTableName + " WHERE KP =
'001'");
+ conn.createStatement().execute("CREATE INDEX " +
globalViewIndexName + " on " +
+ globalViewName + " (KV) " + " INCLUDE (KV2) ASYNC");
+ String tenantId1 = "tenantId1";
+ String tenantId2 = "tenantId2";
+ Properties tenantProps1 = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Properties tenantProps2 = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ tenantProps1.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB,
tenantId1);
+ tenantProps2.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB,
tenantId2);
+
+ //create tenant views and upsert one row, this updates all the
timestamps in the client's cache
+ try (Connection tenantConn1 = cqs.connect(url, tenantProps1);
+ Connection tenantConn2 = cqs.connect(url, tenantProps2)) {
+ tenantConn1.createStatement().execute("CREATE VIEW " +
tenantViewName1 + " AS SELECT * FROM " + globalViewName);
+ tenantConn1.createStatement().execute("UPSERT INTO " +
tenantViewName1 + " (PK, KV, KV2) VALUES " + "('PK1', 'KV', '01')");
+ tenantConn1.commit();
+
+ tenantConn2.createStatement().execute("CREATE VIEW " +
tenantViewName2 + " AS SELECT * FROM " + globalViewName);
+ tenantConn2.createStatement().execute("UPSERT INTO " +
tenantViewName2 + " (PK, KV, KV2) VALUES " + "('PK2', 'KV', '02')");
+ tenantConn2.commit();
+ }
+ // build global view index
+ IndexToolIT.runIndexTool(false, "", globalViewName,
+ globalViewIndexName);
+
+ // query on secondary key should use inherited index for all
tenant views.
+ try (Connection tenantConn1 = cqs.connect(url, tenantProps1);
+ Connection tenantConn2 = cqs.connect(url, tenantProps2)) {
+
+ String query1 = "SELECT KV2 FROM " + tenantViewName1 + "
WHERE KV = 'KV'";
+ String query2 = "SELECT KV2 FROM " + tenantViewName2 + "
WHERE KV = 'KV'";
+
+ ResultSet rs =
tenantConn1.createStatement().executeQuery(query1);
+ assertPlan((PhoenixResultSet) rs, "",
+ tenantViewName1 +
QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR + globalViewIndexName);
+ assertTrue(rs.next());
+ assertEquals("01", rs.getString(1));
+
+ rs = tenantConn2.createStatement().executeQuery(query2);
+ assertPlan((PhoenixResultSet) rs, "",
+ tenantViewName2 +
QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR + globalViewIndexName);
+ assertTrue(rs.next());
+ assertEquals("02", rs.getString(1));
+ }
+ }
+ }
+
+
//Helper methods
+ public static void assertPlan(PhoenixResultSet rs, String schemaName,
String tableName) {
+ PTable table = rs.getContext().getCurrentTable().getTable();
+ assertTrue(table.getSchemaName().getString().equals(schemaName) &&
+ table.getTableName().getString().equals(tableName));
+ }
+
private long getLastDDLTimestamp(String tableName) throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
// Need to use different connection than what is used for creating
table or indexes.
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
index 20bd097e04..99ecb0ed95 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
@@ -3458,6 +3458,49 @@ public class WhereOptimizerTest extends
BaseConnectionlessQueryTest {
}
}
+ /**
+ * Test that tenantId is present in the scan start row key when using an
inherited index on a tenant view.
+ */
+ @Test
+ public void testScanKeyInheritedIndexTenantView() throws Exception {
+ String baseTableName = generateUniqueName();
+ String globalViewName = generateUniqueName();
+ String globalViewIndexName = generateUniqueName();
+ String tenantViewName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // create table, view and view index
+ conn.createStatement().execute("CREATE TABLE " + baseTableName +
+ " (TENANT_ID CHAR(8) NOT NULL, KP CHAR(3) NOT NULL, PK
CHAR(3) NOT NULL, KV CHAR(2), KV2 CHAR(2) " +
+ "CONSTRAINT PK PRIMARY KEY(TENANT_ID, KP, PK))
MULTI_TENANT=true");
+ conn.createStatement().execute("CREATE VIEW " + globalViewName +
+ " AS SELECT * FROM " + baseTableName + " WHERE KP =
'001'");
+ conn.createStatement().execute("CREATE INDEX " +
globalViewIndexName + " on " +
+ globalViewName + " (KV) " + " INCLUDE (KV2)");
+ //create tenant view
+ String tenantId = "tenantId";
+ Properties tenantProps = new Properties();
+ tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ try (Connection tenantConn = DriverManager.getConnection(getUrl(),
tenantProps)) {
+ tenantConn.createStatement().execute("CREATE VIEW " +
tenantViewName + " AS SELECT * FROM " + globalViewName);
+ // query on secondary key
+ String query = "SELECT KV2 FROM " + tenantViewName + " WHERE
KV = 'KV'";
+ PhoenixConnection pconn =
tenantConn.unwrap(PhoenixConnection.class);
+ PhoenixPreparedStatement pstmt = new
PhoenixPreparedStatement(pconn, query);
+ QueryPlan plan = pstmt.compileQuery();
+ plan =
tenantConn.unwrap(PhoenixConnection.class).getQueryServices().getOptimizer().optimize(pstmt,
plan);
+ // optimized query plan should use inherited index
+ assertEquals(tenantViewName + "#" + globalViewIndexName,
plan.getContext().getCurrentTable().getTable().getName().getString());
+ Scan scan = plan.getContext().getScan();
+ PTable viewIndexPTable =
tenantConn.unwrap(PhoenixConnection.class).getTable(globalViewIndexName);
+ // PK of view index [_INDEX_ID, tenant_id, KV, PK]
+ byte[] startRow =
ByteUtil.concat(PLong.INSTANCE.toBytes(viewIndexPTable.getViewIndexId()),
+ PChar.INSTANCE.toBytes(tenantId),
+ PChar.INSTANCE.toBytes("KV"));
+ assertArrayEquals(startRow, scan.getStartRow());
+ }
+ }
+ }
+
private void createBaseTable(String baseTable) throws SQLException {
try (Connection globalConnection =
DriverManager.getConnection(getUrl())) {
