This is an automated email from the ASF dual-hosted git repository.
virajjasani pushed a commit to branch 5.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.3 by this push:
new 67b6012383 PHOENIX-8007 Perf bottleneck - Server side Index metadata
lookup overhead (ADDENDUM) (#2629)
67b6012383 is described below
commit 67b60123835c6a3152c15f012ccb7264eea21a30
Author: Viraj Jasani <[email protected]>
AuthorDate: Thu Oct 8 14:24:35 2026 -0700
PHOENIX-8007 Perf bottleneck - Server side Index metadata lookup overhead
(ADDENDUM) (#2629)
---
.../phoenix/index/PhoenixIndexMetaDataBuilder.java | 24 ++++++-------
.../phoenix/index/PhoenixTransactionalIndexer.java | 5 +--
.../jdbc/HighAvailabilityTestingUtilityIT.java | 39 ++++++++++++++++++++++
3 files changed, 53 insertions(+), 15 deletions(-)
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
index ad3d1b611e..7880fdee9b 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
@@ -58,7 +58,7 @@ public class PhoenixIndexMetaDataBuilder {
* Lazily-computed, cached JDBC URL used to open server-side connections for
index metadata
* lookups.
*/
- private static volatile String serverConnectionUrl;
+ private volatile String serverConnectionUrl;
private final RegionCoprocessorEnvironment env;
@@ -69,13 +69,13 @@ public class PhoenixIndexMetaDataBuilder {
public PhoenixIndexMetaData
getIndexMetaData(MiniBatchOperationInProgress<Mutation> miniBatchOp)
throws IOException {
IndexMetaDataCache indexMetaDataCache =
- getIndexMetaDataCache(env,
miniBatchOp.getOperation(0).getAttributesMap());
+ getIndexMetaDataCache(miniBatchOp.getOperation(0).getAttributesMap());
return new PhoenixIndexMetaData(indexMetaDataCache,
miniBatchOp.getOperation(0).getAttributesMap());
}
- private static IndexMetaDataCache
getIndexMetaDataCache(RegionCoprocessorEnvironment env,
- Map<String, byte[]> attributes) throws IOException {
+ private IndexMetaDataCache getIndexMetaDataCache(Map<String, byte[]>
attributes)
+ throws IOException {
if (attributes == null) {
return IndexMetaDataCache.EMPTY_INDEX_META_DATA_CACHE;
}
@@ -85,7 +85,7 @@ public class PhoenixIndexMetaDataBuilder {
}
boolean useServerMetadata = uuid.length == 0;
if (useServerMetadata) {
- IndexMetaDataCache cacheFromPTable =
getIndexMetaDataCacheFromPTable(env, attributes);
+ IndexMetaDataCache cacheFromPTable =
getIndexMetaDataCacheFromPTable(attributes);
if (cacheFromPTable != null) {
return cacheFromPTable;
}
@@ -127,12 +127,10 @@ public class PhoenixIndexMetaDataBuilder {
/**
* Get IndexMetaDataCache by looking up PTable using table metadata
attributes attached to the
* mutation.
- * @param env RegionCoprocessorEnvironment.
* @param attributes Mutation attributes.
* @return IndexMetaDataCache or null if table metadata not found in
attributes.
*/
- private static IndexMetaDataCache getIndexMetaDataCacheFromPTable(
- RegionCoprocessorEnvironment env, Map<String, byte[]> attributes) {
+ private IndexMetaDataCache getIndexMetaDataCacheFromPTable(Map<String,
byte[]> attributes) {
try {
byte[] schemaBytes =
attributes.get(MutationState.MutationMetadataType.SCHEMA_NAME.toString());
@@ -162,8 +160,9 @@ public class PhoenixIndexMetaDataBuilder {
props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}
QueryUtil.setServerConnection(props);
- try (Connection conn =
DriverManager.getConnection(getServerConnectionUrl(env), props)) {
+ try (Connection conn =
DriverManager.getConnection(getServerConnectionUrl(), props)) {
PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ pconn.getQueryServices().clearUpgradeRequired();
PTable dataTable = pconn.getTable(tenantId, fullTableName);
final List<IndexMaintainer> indexMaintainers =
buildIndexMaintainersFromPTable(dataTable, pconn);
@@ -222,15 +221,14 @@ public class PhoenixIndexMetaDataBuilder {
return indexMaintainers;
}
- private static String getServerConnectionUrl(RegionCoprocessorEnvironment
env)
- throws SQLException {
+ private String getServerConnectionUrl() throws SQLException {
String url = serverConnectionUrl;
if (url == null) {
- synchronized (PhoenixIndexMetaDataBuilder.class) {
+ synchronized (this) {
url = serverConnectionUrl;
if (url == null) {
url = QueryUtil.getConnectionUrl(new Properties(),
env.getConfiguration());
- LOGGER.info("Cached server-side JDBC url for index metadata lookups:
{}", url);
+ LOGGER.debug("Cached server-side JDBC url for index metadata
lookups: {}", url);
serverConnectionUrl = url;
}
}
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index c45b93eb11..db0bdc0faa 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -78,6 +78,7 @@ public class PhoenixTransactionalIndexer implements
RegionObserver, RegionCoproc
new ThreadLocal<BatchMutateContext>();
private PhoenixIndexCodec codec;
+ private PhoenixIndexMetaDataBuilder indexMetaDataBuilder;
private IndexWriter writer;
private boolean stopped;
@@ -91,6 +92,7 @@ public class PhoenixTransactionalIndexer implements
RegionObserver, RegionCoproc
final RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
String serverName = env.getServerName().getServerName();
codec = new PhoenixIndexCodec(env.getConfiguration(),
env.getRegionInfo().getTable().getName());
+ indexMetaDataBuilder = new PhoenixIndexMetaDataBuilder(env);
DelegateRegionCoprocessorEnvironment indexWriterEnv =
new DelegateRegionCoprocessorEnvironment(env,
ConnectionType.INDEX_WRITER_CONNECTION);
// setup the actual index writer
@@ -144,8 +146,7 @@ public class PhoenixTransactionalIndexer implements
RegionObserver, RegionCoproc
return;
}
- PhoenixIndexMetaData indexMetaData =
- new
PhoenixIndexMetaDataBuilder(c.getEnvironment()).getIndexMetaData(miniBatchOp);
+ PhoenixIndexMetaData indexMetaData =
indexMetaDataBuilder.getIndexMetaData(miniBatchOp);
if (
indexMetaData.getClientVersion() >=
MetaDataProtocol.MIN_TX_CLIENT_SIDE_MAINTENANCE
&& !indexMetaData.hasLocalIndexes()
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
index 7453c59903..45003ed17e 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
@@ -24,6 +24,7 @@ import static
org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestin
import static
org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown;
import static
org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static
org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -147,6 +148,44 @@ public class HighAvailabilityTestingUtilityIT {
});
}
+ @Test(timeout = 300000)
+ public void testServerSideIndexMetadataLookupUsesLocalCluster() throws
Exception {
+ try (Connection conn = CLUSTERS.getCluster1Connection(haGroup)) {
+ createIndexedTableAndUpsert(conn, tableName + "_T1", tableName + "_I1");
+ }
+ doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ try (Connection conn = CLUSTERS.getCluster2Connection(haGroup)) {
+ createIndexedTableAndUpsert(conn, tableName + "_T2", tableName +
"_I2");
+ }
+ });
+ doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ try (Connection conn = CLUSTERS.getCluster1Connection(haGroup)) {
+ createIndexedTableAndUpsert(conn, tableName + "_T3", tableName +
"_I3");
+ }
+ });
+ }
+
+ private static void createIndexedTableAndUpsert(Connection conn, String
table, String index)
+ throws Exception {
+ final int numRows = 5;
+ try (Statement stmt = conn.createStatement()) {
+ stmt.execute(String.format(
+ "CREATE TABLE %s (PK VARCHAR PRIMARY KEY, V1 VARCHAR)
UPDATE_CACHE_FREQUENCY=60000",
+ table));
+ stmt.execute(String.format("CREATE INDEX %s ON %s (V1)", index, table));
+ for (int i = 0; i < numRows; i++) {
+ stmt.execute(String.format("UPSERT INTO %s VALUES ('k%d', 'v%d')",
table, i, i));
+ }
+ conn.commit();
+ for (String name : new String[] { table, index }) {
+ try (ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + name))
{
+ assertTrue(rs.next());
+ assertEquals(numRows, rs.getInt(1));
+ }
+ }
+ }
+ }
+
/**
* Test that replication works between HBase cluster(s).
*/
