This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 30ca5d1a196 Flexible URI for connection with DB and new MariaDB driver
(#7895)
30ca5d1a196 is described below
commit 30ca5d1a196f45e43fc585e251591685421a6e23
Author: João Jandre <[email protected]>
AuthorDate: Fri Nov 10 13:27:35 2023 -0300
Flexible URI for connection with DB and new MariaDB driver (#7895)
Co-authored-by: João Jandre <[email protected]>
---
client/conf/db.properties.in | 11 ++
framework/db/pom.xml | 5 +
.../main/java/com/cloud/utils/db/DriverLoader.java | 1 +
.../java/com/cloud/utils/db/TransactionLegacy.java | 153 +++++++++++++++------
.../com/cloud/utils/db/TransactionLegacyTest.java | 117 ++++++++++++++++
5 files changed, 246 insertions(+), 41 deletions(-)
diff --git a/client/conf/db.properties.in b/client/conf/db.properties.in
index 572cfbc1ff2..8f31aff17e6 100644
--- a/client/conf/db.properties.in
+++ b/client/conf/db.properties.in
@@ -29,6 +29,10 @@ db.cloud.driver=@DBDRIVER@
db.cloud.port=3306
db.cloud.name=cloud
+# Connection URI to the database "cloud". When this property is set, only the
following properties will be used along with it: db.cloud.maxActive,
db.cloud.maxIdle, db.cloud.maxWait, db.cloud.username, db.cloud.password,
db.cloud.driver, db.cloud.validationQuery, db.cloud.isolation.level. Other
properties will be ignored.
+db.cloud.uri=
+
+
# CloudStack database tuning parameters
db.cloud.maxActive=250
db.cloud.maxIdle=30
@@ -61,6 +65,10 @@ db.usage.driver=@DBDRIVER@
db.usage.port=3306
db.usage.name=cloud_usage
+# Connection URI to the database "usage". When this property is set, only the
following properties will be used along with it: db.usage.maxActive,
db.cloud.maxIdle, db.cloud.maxWait, db.usage.username, db.usage.password,
db.usage.driver, db.usage.validationQuery, db.usage.isolation.level. Other
properties will be ignored.
+db.usage.uri=
+
+
# usage database tuning parameters
db.usage.maxActive=100
db.usage.maxIdle=30
@@ -79,6 +87,9 @@ db.simulator.maxIdle=30
db.simulator.maxWait=10000
db.simulator.autoReconnect=true
+# Connection URI to the database "simulator". When this property is set, only
the following properties will be used along with it: db.simulator.host,
db.simulator.port, db.simulator.name, db.simulator.autoReconnect. Other
properties will be ignored.
+db.simulator.uri=
+
# High Availability And Cluster Properties
db.ha.enabled=false
diff --git a/framework/db/pom.xml b/framework/db/pom.xml
index c3f99ae61c0..846c9007708 100644
--- a/framework/db/pom.xml
+++ b/framework/db/pom.xml
@@ -53,6 +53,11 @@
<artifactId>cloud-utils</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>3.1.4</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java
b/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java
index 34af85fbcbd..55fc1dbb6ed 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java
@@ -38,6 +38,7 @@ public class DriverLoader {
DRIVERS.put("jdbc:mysql", "com.mysql.cj.jdbc.Driver");
DRIVERS.put("jdbc:postgresql", "org.postgresql.Driver");
DRIVERS.put("jdbc:h2", "org.h2.Driver");
+ DRIVERS.put("jdbc:mariadb", "org.mariadb.jdbc.Driver");
LOADED_DRIVERS = new ArrayList<String>();
}
diff --git
a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
index eb6b09c31f3..df0df60f519 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java
@@ -38,6 +38,7 @@ import
org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnection;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDataSource;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
@@ -1001,7 +1002,7 @@ public class TransactionLegacy implements Closeable {
private static DataSource s_ds;
private static DataSource s_usageDS;
private static DataSource s_simulatorDS;
- private static boolean s_dbHAEnabled;
+ protected static boolean s_dbHAEnabled;
static {
// Initialize with assumed db.properties file
@@ -1032,11 +1033,6 @@ public class TransactionLegacy implements Closeable {
final long cloudMaxWait =
Long.parseLong(dbProps.getProperty("db.cloud.maxWait"));
final String cloudUsername =
dbProps.getProperty("db.cloud.username");
final String cloudPassword =
dbProps.getProperty("db.cloud.password");
- final String cloudHost = dbProps.getProperty("db.cloud.host");
- final String cloudDriver = dbProps.getProperty("db.cloud.driver");
- final int cloudPort =
Integer.parseInt(dbProps.getProperty("db.cloud.port"));
- final String cloudDbName = dbProps.getProperty("db.cloud.name");
- final boolean cloudAutoReconnect =
Boolean.parseBoolean(dbProps.getProperty("db.cloud.autoReconnect"));
final String cloudValidationQuery =
dbProps.getProperty("db.cloud.validationQuery");
final String cloudIsolationLevel =
dbProps.getProperty("db.cloud.isolation.level");
@@ -1059,16 +1055,6 @@ public class TransactionLegacy implements Closeable {
final boolean cloudTestWhileIdle =
Boolean.parseBoolean(dbProps.getProperty("db.cloud.testWhileIdle"));
final long cloudTimeBtwEvictionRunsMillis =
Long.parseLong(dbProps.getProperty("db.cloud.timeBetweenEvictionRunsMillis"));
final long cloudMinEvcitableIdleTimeMillis =
Long.parseLong(dbProps.getProperty("db.cloud.minEvictableIdleTimeMillis"));
- final boolean cloudPoolPreparedStatements =
Boolean.parseBoolean(dbProps.getProperty("db.cloud.poolPreparedStatements"));
- final String url = dbProps.getProperty("db.cloud.url.params");
-
- String cloudDbHAParams = null;
- String cloudReplicas = null;
- if (s_dbHAEnabled) {
- cloudDbHAParams = getDBHAParams("cloud", dbProps);
- cloudReplicas = dbProps.getProperty("db.cloud.replicas");
- s_logger.info("The replicas configured for Cloud Data base
is/are : " + cloudReplicas);
- }
final boolean useSSL =
Boolean.parseBoolean(dbProps.getProperty("db.cloud.useSSL"));
if (useSSL) {
@@ -1078,13 +1064,12 @@ public class TransactionLegacy implements Closeable {
System.setProperty("javax.net.ssl.trustStorePassword",
dbProps.getProperty("db.cloud.trustStorePassword"));
}
- final String cloudConnectionUri = cloudDriver + "://" + cloudHost
+ (s_dbHAEnabled ? "," + cloudReplicas : "") + ":" + cloudPort + "/" +
cloudDbName +
- "?autoReconnect=" + cloudAutoReconnect + (url != null ?
"&" + url : "") + (useSSL ? "&useSSL=true" : "") +
- (s_dbHAEnabled ? "&" + cloudDbHAParams : "") +
(s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : "");
- DriverLoader.loadDriver(cloudDriver);
+ Pair<String, String> cloudUriAndDriver =
getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "cloud");
+
+ DriverLoader.loadDriver(cloudUriAndDriver.second());
// Default Data Source for CloudStack
- s_ds = createDataSource(cloudConnectionUri, cloudUsername,
cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait,
+ s_ds = createDataSource(cloudUriAndDriver.first(), cloudUsername,
cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait,
cloudTimeBtwEvictionRunsMillis,
cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle, cloudTestOnBorrow,
cloudValidationQuery, isolationLevel);
@@ -1094,20 +1079,13 @@ public class TransactionLegacy implements Closeable {
final long usageMaxWait =
Long.parseLong(dbProps.getProperty("db.usage.maxWait"));
final String usageUsername =
dbProps.getProperty("db.usage.username");
final String usagePassword =
dbProps.getProperty("db.usage.password");
- final String usageHost = dbProps.getProperty("db.usage.host");
- final String usageDriver = dbProps.getProperty("db.usage.driver");
- final int usagePort =
Integer.parseInt(dbProps.getProperty("db.usage.port"));
- final String usageDbName = dbProps.getProperty("db.usage.name");
- final boolean usageAutoReconnect =
Boolean.parseBoolean(dbProps.getProperty("db.usage.autoReconnect"));
- final String usageUrl = dbProps.getProperty("db.usage.url.params");
-
- final String usageConnectionUri = usageDriver + "://" + usageHost
+ (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.replicas") : "") + ":" +
usagePort +
- "/" + usageDbName + "?autoReconnect=" + usageAutoReconnect
+ (usageUrl != null ? "&" + usageUrl : "") +
- (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) :
"") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : "");
- DriverLoader.loadDriver(usageDriver);
+
+ Pair<String, String> usageUriAndDriver =
getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "usage");
+
+ DriverLoader.loadDriver(usageUriAndDriver.second());
// Data Source for usage server
- s_usageDS = createDataSource(usageConnectionUri, usageUsername,
usagePassword,
+ s_usageDS = createDataSource(usageUriAndDriver.first(),
usageUsername, usagePassword,
usageMaxActive, usageMaxIdle, usageMaxWait, null, null,
null, null,
null, isolationLevel);
@@ -1118,14 +1096,28 @@ public class TransactionLegacy implements Closeable {
final long simulatorMaxWait =
Long.parseLong(dbProps.getProperty("db.simulator.maxWait"));
final String simulatorUsername =
dbProps.getProperty("db.simulator.username");
final String simulatorPassword =
dbProps.getProperty("db.simulator.password");
- final String simulatorHost =
dbProps.getProperty("db.simulator.host");
- final String simulatorDriver =
dbProps.getProperty("db.simulator.driver");
- final int simulatorPort =
Integer.parseInt(dbProps.getProperty("db.simulator.port"));
- final String simulatorDbName =
dbProps.getProperty("db.simulator.name");
- final boolean simulatorAutoReconnect =
Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect"));
-
- final String simulatorConnectionUri = simulatorDriver + "://"
+ simulatorHost + ":" + simulatorPort + "/" + simulatorDbName +
"?autoReconnect=" +
- simulatorAutoReconnect;
+
+ String simulatorDriver;
+ String simulatorConnectionUri;
+ String simulatorUri = dbProps.getProperty("db.simulator.uri");
+
+ if (StringUtils.isEmpty(simulatorUri)) {
+ simulatorDriver =
dbProps.getProperty("db.simulator.driver");
+ final int simulatorPort =
Integer.parseInt(dbProps.getProperty("db.simulator.port"));
+ final String simulatorDbName =
dbProps.getProperty("db.simulator.name");
+ final boolean simulatorAutoReconnect =
Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect"));
+ final String simulatorHost =
dbProps.getProperty("db.simulator.host");
+
+ simulatorConnectionUri = simulatorDriver + "://" +
simulatorHost + ":" + simulatorPort + "/" + simulatorDbName + "?autoReconnect="
+
+ simulatorAutoReconnect;
+ } else {
+ s_logger.warn("db.simulator.uri was set, ignoring the
following properties on db.properties: [db.simulator.driver, db.simulator.host,
db.simulator.port, "
+ + "db.simulator.name,
db.simulator.autoReconnect].");
+ String[] splitUri = simulatorUri.split(":");
+ simulatorDriver = String.format("%s:%s", splitUri[0],
splitUri[1]);
+ simulatorConnectionUri = simulatorUri;
+ }
+
DriverLoader.loadDriver(simulatorDriver);
s_simulatorDS = createDataSource(simulatorConnectionUri,
simulatorUsername, simulatorPassword,
@@ -1143,6 +1135,85 @@ public class TransactionLegacy implements Closeable {
}
}
+ protected static Pair<String, String> getConnectionUriAndDriver(Properties
dbProps, String loadBalanceStrategy, boolean useSSL, String schema) {
+ String connectionUri;
+ String driver;
+ String propertyUri = dbProps.getProperty(String.format("db.%s.uri",
schema));
+
+ if (StringUtils.isEmpty(propertyUri)) {
+ driver = dbProps.getProperty(String.format("db.%s.driver",
schema));
+ connectionUri = getPropertiesAndBuildConnectionUri(dbProps,
loadBalanceStrategy, driver, useSSL, schema);
+ } else {
+ s_logger.warn(String.format("db.%s.uri was set, ignoring the
following properties for schema %s of db.properties: [host, port, name, driver,
autoReconnect, url.params,"
+ + " replicas, ha.loadBalanceStrategy, ha.enable,
failOverReadOnly, reconnectAtTxEnd, autoReconnectForPools,
secondsBeforeRetrySource, queriesBeforeRetrySource, "
+ + "initialTimeout].", schema, schema));
+
+ String[] splitUri = propertyUri.split(":");
+ driver = String.format("%s:%s", splitUri[0], splitUri[1]);
+
+ connectionUri = propertyUri;
+ }
+ s_logger.info(String.format("Using the following URI to connect to %s
database [%s].", schema, connectionUri));
+ return new Pair<>(connectionUri, driver);
+ }
+
+ protected static String getPropertiesAndBuildConnectionUri(Properties
dbProps, String loadBalanceStrategy, String driver, boolean useSSL, String
schema) {
+ String host = dbProps.getProperty(String.format("db.%s.host", schema));
+ int port =
Integer.parseInt(dbProps.getProperty(String.format("db.%s.port", schema)));
+ String dbName = dbProps.getProperty(String.format("db.%s.name",
schema));
+ boolean autoReconnect =
Boolean.parseBoolean(dbProps.getProperty(String.format("db.%s.autoReconnect",
schema)));
+ String urlParams =
dbProps.getProperty(String.format("db.%s.url.params", schema));
+
+ String replicas = null;
+ String dbHaParams = null;
+ if (s_dbHAEnabled) {
+ dbHaParams = getDBHAParams(schema, dbProps);
+ replicas = dbProps.getProperty(String.format("db.%s.replicas",
schema));
+ s_logger.info(String.format("The replicas configured for %s data
base are %s.", schema, replicas));
+ }
+
+ return buildConnectionUri(loadBalanceStrategy, driver, useSSL, host,
replicas, port, dbName, autoReconnect, urlParams, dbHaParams);
+ }
+
+ protected static String buildConnectionUri(String loadBalanceStrategy,
String driver, boolean useSSL, String host, String replicas, int port, String
dbName, boolean autoReconnect,
+ String urlParams, String dbHaParams) {
+
+ StringBuilder connectionUri = new StringBuilder();
+ connectionUri.append(driver);
+ connectionUri.append("://");
+ connectionUri.append(host);
+
+ if (s_dbHAEnabled) {
+ connectionUri.append(",");
+ connectionUri.append(replicas);
+ }
+
+ connectionUri.append(":");
+ connectionUri.append(port);
+ connectionUri.append("/");
+ connectionUri.append(dbName);
+ connectionUri.append("?autoReconnect=");
+ connectionUri.append(autoReconnect);
+
+ if (urlParams != null) {
+ connectionUri.append("&");
+ connectionUri.append(urlParams);
+ }
+
+ if (useSSL) {
+ connectionUri.append("&useSSL=true");
+ }
+
+ if (s_dbHAEnabled) {
+ connectionUri.append("&");
+ connectionUri.append(dbHaParams);
+ connectionUri.append("&loadBalanceStrategy=");
+ connectionUri.append(loadBalanceStrategy);
+ }
+
+ return connectionUri.toString();
+ }
+
/**
* Creates a data source
*/
diff --git
a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
new file mode 100644
index 00000000000..2e0af6fa186
--- /dev/null
+++ b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.utils.db;
+
+import com.cloud.utils.Pair;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Properties;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TransactionLegacyTest {
+
+ Properties properties;
+
+ @Before
+ public void setup(){
+ properties = new Properties();
+ properties.setProperty("db.cloud.host", "host");
+ properties.setProperty("db.cloud.port", "5555");
+ properties.setProperty("db.cloud.name", "name");
+ properties.setProperty("db.cloud.autoReconnect", "false");
+ properties.setProperty("db.cloud.url.params", "someParams");
+ TransactionLegacy.s_dbHAEnabled = false;
+ }
+ @Test
+ public void getConnectionUriAndDriverTestWithoutUri() {
+ properties.setProperty("db.cloud.uri", "");
+ properties.setProperty("db.cloud.driver", "driver");
+
+ Pair<String, String> result =
TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+
Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams",
result.first());
+ Assert.assertEquals("driver", result.second());
+ }
+
+ @Test
+ public void getConnectionUriAndDriverTestWithUri() {
+ properties.setProperty("db.cloud.uri", "jdbc:driver:myFavoriteUri");
+
+ Pair<String, String> result =
TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+ Assert.assertEquals("jdbc:driver:myFavoriteUri", result.first());
+ Assert.assertEquals("jdbc:driver", result.second());
+ }
+
+ @Test
+ public void getPropertiesAndBuildConnectionUriTestDbHaDisabled() {
+ String result =
TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat",
"driver", true, "cloud");
+
+
Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true",
result);
+ }
+
+ @Test
+ public void getPropertiesAndBuildConnectionUriTestDbHaEnabled() {
+ TransactionLegacy.s_dbHAEnabled = true;
+ properties.setProperty("db.cloud.failOverReadOnly", "true");
+ properties.setProperty("db.cloud.reconnectAtTxEnd", "false");
+ properties.setProperty("db.cloud.autoReconnectForPools", "true");
+ properties.setProperty("db.cloud.secondsBeforeRetrySource", "25");
+ properties.setProperty("db.cloud.queriesBeforeRetrySource", "105");
+ properties.setProperty("db.cloud.initialTimeout", "1000");
+ properties.setProperty("db.cloud.replicas", "second_host");
+
+ String result =
TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat",
"driver", true, "cloud");
+
+
Assert.assertEquals("driver://host,second_host:5555/name?autoReconnect=false&someParams&useSSL=true&failOverReadOnly=true&reconnectAtTxEnd=false&autoReconnectFor"
+ +
"Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat",
result);
+ }
+
+ @Test
+ public void buildConnectionUriTestDbHaDisabled() {
+ String result = TransactionLegacy.buildConnectionUri(null, "driver",
false, "host", null, 5555, "cloud", false, null, null);
+
+ Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false",
result);
+ }
+
+ @Test
+ public void buildConnectionUriTestDbHaEnabled() {
+ TransactionLegacy.s_dbHAEnabled = true;
+
+ String result = TransactionLegacy.buildConnectionUri("strat",
"driver", false, "host", "second_host", 5555, "cloud", false, null,
"dbHaParams");
+
+
Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat",
result);
+ }
+
+ @Test
+ public void buildConnectionUriTestUrlParamsNotNull() {
+ String result = TransactionLegacy.buildConnectionUri(null, "driver",
false, "host", null, 5555, "cloud", false, "urlParams", null);
+
+
Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams",
result);
+ }
+
+ @Test
+ public void buildConnectionUriTestUseSslTrue() {
+ String result = TransactionLegacy.buildConnectionUri(null, "driver",
true, "host", null, 5555, "cloud", false, null, null);
+
+
Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true",
result);
+ }
+}
