This is an automated email from the ASF dual-hosted git repository.
dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 17b46d256a0 HIVE-30218: Eliminate redundant txn isolation round-trips
in Metastore and upgrade HikariCP to 7.1.0 (#6917)
17b46d256a0 is described below
commit 17b46d256a0866b2cf4fad20a54ac77bf9707b56
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Mon Nov 2 13:20:22 2026 +0530
HIVE-30218: Eliminate redundant txn isolation round-trips in Metastore and
upgrade HikariCP to 7.1.0 (#6917)
---
pom.xml | 2 +-
standalone-metastore/metastore-common/pom.xml | 4 --
.../datasource/HikariCPDataSourceProvider.java | 13 ++++++
.../datasource/TestDataSourceProviderFactory.java | 51 ++++++++++++++++++++++
standalone-metastore/pom.xml | 2 +-
5 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/pom.xml b/pom.xml
index abdcf5e9e8c..18c1867e57a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,7 @@
<graalvm.version>25.1.3</graalvm.version>
<!-- required for logging test to avoid including hbase which pulls
disruptor transitively -->
<disruptor.version>3.3.7</disruptor.version>
- <hikaricp.version>4.0.3</hikaricp.version>
+ <hikaricp.version>7.1.0</hikaricp.version>
<!-- httpcomponents are not always in version sync -->
<httpcomponents.client.version>4.5.13</httpcomponents.client.version>
<httpcomponents.core.version>4.4.13</httpcomponents.core.version>
diff --git a/standalone-metastore/metastore-common/pom.xml
b/standalone-metastore/metastore-common/pom.xml
index fba1512a2ab..f063cb9d999 100644
--- a/standalone-metastore/metastore-common/pom.xml
+++ b/standalone-metastore/metastore-common/pom.xml
@@ -94,10 +94,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
- <dependency>
- <groupId>com.zaxxer</groupId>
- <artifactId>HikariCP</artifactId>
- </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
index 9f3f0f10f99..ce8765d8cb3 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
@@ -39,6 +39,8 @@ public class HikariCPDataSourceProvider implements
DataSourceProvider {
private static final Logger LOG =
LoggerFactory.getLogger(HikariCPDataSourceProvider.class);
static final String HIKARI = "hikaricp";
+ static final String TRANSACTION_ISOLATION = "transactionIsolation";
+ static final String TRANSACTION_READ_COMMITTED =
"TRANSACTION_READ_COMMITTED";
@Override
public DataSource createPool(Configuration hdpConfig, int maxPoolSize)
throws SQLException {
@@ -76,6 +78,17 @@ public DataSource createPool(Configuration hdpConfig, int
maxPoolSize) throws SQ
config.setMinimumIdle(Math.min(maxPoolSize, minimumIdle));
}
+ // Every consumer of these pools runs at READ COMMITTED:
TransactionContextManager requests
+ // Isolation.READ_COMMITTED for each TxnHandler transaction and
DataNucleus applies
+ // datanucleus.transactionIsolation (default read-committed) to every
connection it borrows.
+ // Making it the pool default lets HikariCP set it once per physical
connection, instead of
+ // Spring/DataNucleus changing and restoring it on every transaction,
which costs two extra
+ // round trips per transaction on databases whose default differs
(MySQL/MariaDB default to
+ // REPEATABLE READ). An explicit hikaricp.transactionIsolation still takes
precedence.
+ if (!properties.containsKey(TRANSACTION_ISOLATION)) {
+ config.setTransactionIsolation(TRANSACTION_READ_COMMITTED);
+ }
+
DataSourceProvider.preparePool(hdpConfig, s ->
config.setConnectionInitSql(s),
entry -> config.addDataSourceProperty(entry.getKey(),
entry.getValue()));
return new HikariDataSource(initMetrics(config));
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
index 42c8b8f5e40..e09b939935e 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
@@ -77,6 +77,9 @@ public void testDefaultHikariCpProperties() throws
SQLException {
Assert.assertEquals(1800000L, hds.getMaxLifetime());
Assert.assertEquals(0L, hds.getLeakDetectionThreshold());
Assert.assertEquals(1L, hds.getInitializationFailTimeout());
+ // HikariCP >= 6.2.1 enables a 2 minute keepalive by default; idle
connections are validated
+ // periodically without resetting their idle time, so idleTimeout based
retirement still applies
+ Assert.assertEquals(120000L, hds.getKeepaliveTime());
}
@Test
@@ -171,6 +174,54 @@ public void testSetHikariCpBooleanProperty() throws
SQLException {
Assert.assertEquals(false, ((HikariDataSource)ds).isAllowPoolSuspension());
}
+ @Test
+ public void testHikariCpDefaultTransactionIsolation() throws SQLException {
+ MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE,
HikariCPDataSourceProvider.HIKARI);
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout",
"-1");
+
+ DataSourceProvider dsp =
DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ // Every metastore consumer of the pool runs at READ COMMITTED, so the
pool defaults to it
+ Assert.assertEquals(HikariCPDataSourceProvider.TRANSACTION_READ_COMMITTED,
+ ((HikariDataSource) ds).getTransactionIsolation());
+ }
+
+ @Test
+ public void testSetHikariCpTransactionIsolation() throws SQLException {
+ MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE,
HikariCPDataSourceProvider.HIKARI);
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".transactionIsolation",
"TRANSACTION_SERIALIZABLE");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout",
"-1");
+
+ DataSourceProvider dsp =
DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ // An explicit hikaricp.transactionIsolation overrides the READ COMMITTED
default
+ Assert.assertEquals("TRANSACTION_SERIALIZABLE", ((HikariDataSource)
ds).getTransactionIsolation());
+ }
+
+ @Test
+ public void testSetHikariCpDurationProperty() throws SQLException {
+ MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE,
HikariCPDataSourceProvider.HIKARI);
+ // HikariCP accepts duration strings (ms, s, m, h, d) for time based
properties
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".idleTimeout", "1m");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".maxLifetime", "45m");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout",
"-1");
+
+ DataSourceProvider dsp =
DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ HikariDataSource hds = (HikariDataSource) ds;
+ Assert.assertEquals(60000L, hds.getIdleTimeout());
+ Assert.assertEquals(2700000L, hds.getMaxLifetime());
+ }
+
@Test
public void testCreateDbCpDataSource() throws SQLException {
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 1c07f40e591..282c3e9c2dd 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -92,7 +92,7 @@
<dropwizard.version>3.1.0</dropwizard.version>
<guava.version>22.0</guava.version>
<hadoop.version>3.4.2</hadoop.version>
- <hikaricp.version>4.0.3</hikaricp.version>
+ <hikaricp.version>7.1.0</hikaricp.version>
<iceberg.version>1.12.0</iceberg.version>
<jexl.version>3.7.0</jexl.version>
<jackson.version>2.21.5</jackson.version>