This is an automated email from the ASF dual-hosted git repository.
jianglongtao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new cda2164ae79 Add convert db-discovery distsql and test case (#20707)
cda2164ae79 is described below
commit cda2164ae79b4a0cf41e485464db4690c36fdfe8
Author: Xin Huang <[email protected]>
AuthorDate: Fri Sep 2 10:10:28 2022 +0800
Add convert db-discovery distsql and test case (#20707)
* rebase conflicts fixed
* rebase conflict in yaml fixed
* conflict fixed
* QueryableRALStatementAssert.java bug fixed
* indentation bug fixed
* convertyamlconfigurationhandler conflict fixed
* db-discovert constants conflict fixed
---
.../common/constant/DistSQLScriptConstants.java | 28 +++++--
.../queryable/ConvertYamlConfigurationHandler.java | 88 +++++++++++++++++++++-
.../ConvertYamlConfigurationHandlerTest.java | 16 +++-
.../conf/convert/config-database-discovery.yaml | 67 ++++++++++++++++
.../expected/convert-database-discovery.yaml | 40 ++++++++++
5 files changed, 230 insertions(+), 9 deletions(-)
diff --git
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/common/constant/DistSQLScriptConstants.java
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/common/constant/DistSQLScriptConstants.java
index 4eb1b890eb7..164936e5896 100644
---
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/common/constant/DistSQLScriptConstants.java
+++
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/common/constant/DistSQLScriptConstants.java
@@ -25,13 +25,15 @@ import lombok.NoArgsConstructor;
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class DistSQLScriptConstants {
-
+
public static final String RESOURCE_DB = "resource_db";
public static final String SHARDING_DB = "sharding_db";
public static final String READWRITE_SPLITTING_DB =
"readwrite_splitting_db";
-
+
+ public static final String DATABASE_DISCOVERY_DB = "database_discovery_db";
+
public static final String STANDARD = "standard";
public static final String COMPLEX = "complex";
@@ -134,10 +136,26 @@ public final class DistSQLScriptConstants {
+ "%s"
+ System.lineSeparator()
+ ")";
+
+ public static final String READWRITE_SPLITTING_TYPE = "TYPE(NAME=\"%s\")";
- public static final String TYPE = "TYPE(NAME=\"%s\")";
-
- public static final String TYPE_PROPERTIES = "TYPE(NAME=\"%s\",
PROPERTIES(%s))";
+ public static final String READWRITE_SPLITTING_TYPE_PROPERTIES =
"TYPE(NAME=\"%s\", PROPERTIES(%s))";
public static final String READ_RESOURCE = "%s";
+
+ public static final String CREATE_DB_DISCOVERY = "CREATE DB_DISCOVERY
RULE";
+
+ public static final String DB_DISCOVERY = " %s ("
+ + System.lineSeparator()
+ + "RESOURCES(%s),"
+ + System.lineSeparator()
+ + "%s,"
+ + System.lineSeparator()
+ + "HEARTBEAT(PROPERTIES(%s))"
+ + System.lineSeparator()
+ + ")";
+
+ public static final String DB_DISCOVERY_TYPE = "TYPE(NAME=\'%s\',
PROPERTIES(%s))";
+
+ public static final String DB_DISCOVERY_PROPERTY = "\'%s\'=\'%s\'";
}
diff --git
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandler.java
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandler.java
index 29742f269ba..66938ab49b4 100644
---
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandler.java
+++
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandler.java
@@ -20,6 +20,8 @@ package
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable;
import com.google.common.base.Preconditions;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.commons.lang3.StringUtils;
+import
org.apache.shardingsphere.dbdiscovery.yaml.config.rule.YamlDatabaseDiscoveryDataSourceRuleConfiguration;
+import
org.apache.shardingsphere.dbdiscovery.yaml.config.rule.YamlDatabaseDiscoveryHeartBeatConfiguration;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ConvertYamlConfigurationStatement;
import org.apache.shardingsphere.infra.config.algorithm.AlgorithmConfiguration;
import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
@@ -48,6 +50,7 @@ import
org.apache.shardingsphere.sharding.api.config.strategy.sharding.StandardS
import
org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration;
import
org.apache.shardingsphere.sharding.yaml.swapper.YamlShardingRuleConfigurationSwapper;
import
org.apache.shardingsphere.readwritesplitting.yaml.config.YamlReadwriteSplittingRuleConfiguration;
+import
org.apache.shardingsphere.dbdiscovery.yaml.config.YamlDatabaseDiscoveryRuleConfiguration;
import java.io.File;
import java.io.IOException;
@@ -97,6 +100,9 @@ public final class ConvertYamlConfigurationHandler extends
QueryableRALBackendHa
case DistSQLScriptConstants.READWRITE_SPLITTING_DB:
addReadWriteSplittingDistSQL(yamlConfig, result);
break;
+ case DistSQLScriptConstants.DATABASE_DISCOVERY_DB:
+ addDatabaseDiscoveryDistSQL(yamlConfig, result);
+ break;
default:
break;
}
@@ -119,6 +125,12 @@ public final class ConvertYamlConfigurationHandler extends
QueryableRALBackendHa
appendResources(yamlConfig.getDataSources(), result);
appendReadWriteSplittingRules(yamlConfig.getRules(), result);
}
+
+ private void addDatabaseDiscoveryDistSQL(final
YamlProxyDatabaseConfiguration yamlConfig, final StringBuilder result) {
+ appendDatabase(yamlConfig.getDatabaseName(), result);
+ appendResources(yamlConfig.getDataSources(), result);
+ appendDatabaseDiscoveryRules(yamlConfig.getRules(), result);
+ }
private void appendDatabase(final String databaseName, final StringBuilder
result) {
result.append(String.format(DistSQLScriptConstants.CREATE_DATABASE,
databaseName)).append(System.lineSeparator());
@@ -358,7 +370,7 @@ public final class ConvertYamlConfigurationHandler extends
QueryableRALBackendHa
StringBuilder result = new StringBuilder();
String loadBalancerProperties = "";
if (loadBalancers.getValue().getProps().isEmpty()) {
- result.append(String.format(DistSQLScriptConstants.TYPE,
loadBalancers.getValue().getType()));
+
result.append(String.format(DistSQLScriptConstants.READWRITE_SPLITTING_TYPE,
loadBalancers.getValue().getType()));
} else {
Iterator<Entry<Object, Object>> iterator =
loadBalancers.getValue().getProps().entrySet().iterator();
while (iterator.hasNext()) {
@@ -367,7 +379,7 @@ public final class ConvertYamlConfigurationHandler extends
QueryableRALBackendHa
loadBalancerProperties =
appendLoadBalancerProperties(loadBalancers.getValue().getProps());
}
}
-
result.append(String.format(DistSQLScriptConstants.TYPE_PROPERTIES,
loadBalancers.getValue().getType(), loadBalancerProperties));
+
result.append(String.format(DistSQLScriptConstants.READWRITE_SPLITTING_TYPE_PROPERTIES,
loadBalancers.getValue().getType(), loadBalancerProperties));
}
return result.toString();
}
@@ -384,4 +396,76 @@ public final class ConvertYamlConfigurationHandler extends
QueryableRALBackendHa
}
return result.toString();
}
+
+ private void appendDatabaseDiscoveryRules(final
Collection<YamlRuleConfiguration> ruleConfigs, final StringBuilder result) {
+ if (ruleConfigs.isEmpty()) {
+ return;
+ }
+ result.append(DistSQLScriptConstants.CREATE_DB_DISCOVERY);
+ for (YamlRuleConfiguration ruleConfig: ruleConfigs) {
+ Iterator<Entry<String,
YamlDatabaseDiscoveryDataSourceRuleConfiguration>> dataSourcesIter =
((YamlDatabaseDiscoveryRuleConfiguration)
ruleConfig).getDataSources().entrySet().iterator();
+ while (dataSourcesIter.hasNext()) {
+ Entry<String,
YamlDatabaseDiscoveryDataSourceRuleConfiguration> entry =
dataSourcesIter.next();
+ String databaseDiscoveryName = entry.getKey();
+ String databaseSourceNames =
getDatabaseDiscoveryResources(entry.getValue().getDataSourceNames());
+ String databaseType =
getDatabaseDiscoveryType(entry.getValue().getDiscoveryTypeName(), ruleConfig);
+ String databaseHeartbeatProperty =
getDatabaseDiscoveryHeartbeat(entry.getValue().getDiscoveryHeartbeatName(),
ruleConfig);
+
result.append(String.format(DistSQLScriptConstants.DB_DISCOVERY,
databaseDiscoveryName, databaseSourceNames, databaseType,
databaseHeartbeatProperty));
+ if (dataSourcesIter.hasNext()) {
+ result.append(DistSQLScriptConstants.COMMA);
+ }
+ }
+
result.append(DistSQLScriptConstants.SEMI).append(System.lineSeparator());
+ }
+ }
+
+ private String getDatabaseDiscoveryResources(final Collection<String>
databaseDiscoveryNames) {
+ StringBuilder result = new StringBuilder();
+ Iterator<String> iterator = databaseDiscoveryNames.iterator();
+ while (iterator.hasNext()) {
+ String databaseDiscoveryName = iterator.next();
+ result.append(databaseDiscoveryName);
+ if (iterator.hasNext()) {
+ result.append(DistSQLScriptConstants.COMMA);
+ }
+ }
+ return result.toString();
+ }
+
+ private String getDatabaseDiscoveryHeartbeat(final String
discoveryHeartbeatName, final YamlRuleConfiguration ruleConfig) {
+ StringBuilder result = new StringBuilder();
+ Iterator<Entry<String, YamlDatabaseDiscoveryHeartBeatConfiguration>>
discoveryHeartbeatsIter =
+ ((YamlDatabaseDiscoveryRuleConfiguration)
ruleConfig).getDiscoveryHeartbeats().entrySet().iterator();
+ while (discoveryHeartbeatsIter.hasNext()) {
+ Entry<String, YamlDatabaseDiscoveryHeartBeatConfiguration> entry =
discoveryHeartbeatsIter.next();
+ if (entry.getKey().equals(discoveryHeartbeatName)) {
+ getDatabaseDiscoveryProperties(entry.getValue().getProps(),
result);
+ }
+ }
+ return result.toString();
+ }
+
+ private String getDatabaseDiscoveryType(final String discoveryTypeName,
final YamlRuleConfiguration ruleConfig) {
+ StringBuilder result = new StringBuilder();
+ StringBuilder properties = new StringBuilder();
+ Iterator<Entry<String, YamlAlgorithmConfiguration>> discoveryTypesIter
= ((YamlDatabaseDiscoveryRuleConfiguration)
ruleConfig).getDiscoveryTypes().entrySet().iterator();
+ while (discoveryTypesIter.hasNext()) {
+ Entry<String, YamlAlgorithmConfiguration> entry =
discoveryTypesIter.next();
+ if (entry.getKey().equals(discoveryTypeName)) {
+ getDatabaseDiscoveryProperties(entry.getValue().getProps(),
properties);
+ String typeName = entry.getValue().getType();
+ String typePros = properties.toString();
+
result.append(String.format(DistSQLScriptConstants.DB_DISCOVERY_TYPE, typeName,
typePros));
+ }
+ }
+ return result.toString();
+ }
+
+ private void getDatabaseDiscoveryProperties(final Properties
heartbeatProperties, final StringBuilder result) {
+ Iterator<Entry<Object, Object>> iterator =
heartbeatProperties.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Entry<Object, Object> entry = iterator.next();
+
result.append(String.format(DistSQLScriptConstants.DB_DISCOVERY_PROPERTY,
entry.getKey(), entry.getValue()));
+ }
+ }
}
diff --git
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandlerTest.java
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandlerTest.java
index 04749af37c1..bb0074b81fe 100644
---
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandlerTest.java
+++
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ConvertYamlConfigurationHandlerTest.java
@@ -51,26 +51,33 @@ public final class ConvertYamlConfigurationHandlerTest
extends ProxyContextResto
private final String shardingFilePath =
"/conf/convert/config-sharding.yaml";
private final String readWriteSplittingFilePath =
"/conf/convert/config-readwrite-splitting.yaml";
+
+ private final String databaseDiscoveryFilePath =
"/conf/convert/config-database-discovery.yaml";
private final String resourceExpectedFilePath =
"/expected/convert-add-resource.yaml";
private final String shardingExpectedFilePath =
"/expected/convert-create-sharding.yaml";
private final String readWriteSplittingExpectedFilePath =
"/expected/convert-readwrite-splitting.yaml";
+
+ private final String databaseDiscoveryExpectedFilePath =
"/expected/convert-database-discovery.yaml";
private final String resource = "resource";
private final String sharding = "sharding";
private final String readWriteSplitting = "readWriteSplitting";
+
+ private final String databaseDiscovery = "databaseDiscovery";
- private final Map<String, String> featureMap = new HashMap<>(3, 1);
+ private final Map<String, String> featureMap = new HashMap<>(4, 1);
@Before
public void setup() {
featureMap.put(resource, resourceFilePath);
featureMap.put(sharding, shardingFilePath);
featureMap.put(readWriteSplitting, readWriteSplittingFilePath);
+ featureMap.put(databaseDiscovery, databaseDiscoveryFilePath);
}
@Before
@@ -93,7 +100,12 @@ public final class ConvertYamlConfigurationHandlerTest
extends ProxyContextResto
public void assertExecuteWithReadWriteSplitting() throws SQLException {
assertExecute(readWriteSplitting, readWriteSplittingExpectedFilePath);
}
-
+
+ @Test
+ public void assertExecuteWithDatabaseDiscovery() throws SQLException {
+ assertExecute(databaseDiscovery, databaseDiscoveryExpectedFilePath);
+ }
+
public void assertExecute(final String type, final String
expectedFilePath) throws SQLException {
ConvertYamlConfigurationHandler handler = new
ConvertYamlConfigurationHandler();
handler.init(new
ConvertYamlConfigurationStatement(Objects.requireNonNull(ConvertYamlConfigurationHandlerTest.class.getResource(featureMap.get(type))).getPath()),
diff --git
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/conf/convert/config-database-discovery.yaml
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/conf/convert/config-database-discovery.yaml
new file mode 100644
index 00000000000..dcb5417fe4b
--- /dev/null
+++
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/conf/convert/config-database-discovery.yaml
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+databaseName: database_discovery_db
+
+dataSources:
+ ds_0:
+ url:
jdbc:mysql://127.0.0.1:3306/demo_primary_ds?serverTimezone=UTC&useSSL=false
+ username: root
+ password: 12345678
+ connectionTimeoutMilliseconds: 3000
+ idleTimeoutMilliseconds: 60000
+ maxLifetimeMilliseconds: 1800000
+ maxPoolSize: 50
+ minPoolSize: 1
+ ds_1:
+ url:
jdbc:mysql://127.0.0.1:3306/demo_replica_ds_0?serverTimezone=UTC&useSSL=false
+ username: root
+ password: 12345678
+ connectionTimeoutMilliseconds: 3000
+ idleTimeoutMilliseconds: 60000
+ maxLifetimeMilliseconds: 1800000
+ maxPoolSize: 50
+ minPoolSize: 1
+ ds_2:
+ url:
jdbc:mysql://127.0.0.1:3306/demo_replica_ds_1?serverTimezone=UTC&useSSL=false
+ username: root
+ password: 12345678
+ connectionTimeoutMilliseconds: 3000
+ idleTimeoutMilliseconds: 60000
+ maxLifetimeMilliseconds: 1800000
+ maxPoolSize: 50
+ minPoolSize: 1
+
+rules:
+ - !DB_DISCOVERY
+ dataSources:
+ readwrite_ds:
+ dataSourceNames:
+ - ds_0
+ - ds_1
+ - ds_2
+ discoveryHeartbeatName: mgr-heartbeat
+ discoveryTypeName: mgr
+ discoveryHeartbeats:
+ mgr-heartbeat:
+ props:
+ keep-alive-cron: '0/5 * * * * ?'
+ discoveryTypes:
+ mgr:
+ type: MySQL.MGR
+ props:
+ group-name: 92504d5b-6dec-11e8-91ea-246e9612aaf1
diff --git
a/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/expected/convert-database-discovery.yaml
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/expected/convert-database-discovery.yaml
new file mode 100644
index 00000000000..aeac2b2b83d
--- /dev/null
+++
b/shardingsphere-proxy/shardingsphere-proxy-backend/src/test/resources/expected/convert-database-discovery.yaml
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+CREATE DATABASE database_discovery_db;
+USE database_discovery_db;
+ADD RESOURCE ds_0 (
+URL="jdbc:mysql://127.0.0.1:3306/demo_primary_ds?serverTimezone=UTC&useSSL=false",
+USER="root",
+PASSWORD="12345678",
+PROPERTIES("minPoolSize"="1","connectionTimeoutMilliseconds"="3000","maxLifetimeMilliseconds"="1800000","idleTimeoutMilliseconds"="60000","maxPoolSize"="50")
+), ds_1 (
+URL="jdbc:mysql://127.0.0.1:3306/demo_replica_ds_0?serverTimezone=UTC&useSSL=false",
+USER="root",
+PASSWORD="12345678",
+PROPERTIES("minPoolSize"="1","connectionTimeoutMilliseconds"="3000","maxLifetimeMilliseconds"="1800000","idleTimeoutMilliseconds"="60000","maxPoolSize"="50")
+), ds_2 (
+URL="jdbc:mysql://127.0.0.1:3306/demo_replica_ds_1?serverTimezone=UTC&useSSL=false",
+USER="root",
+PASSWORD="12345678",
+PROPERTIES("minPoolSize"="1","connectionTimeoutMilliseconds"="3000","maxLifetimeMilliseconds"="1800000","idleTimeoutMilliseconds"="60000","maxPoolSize"="50")
+);
+CREATE DB_DISCOVERY RULE readwrite_ds (
+RESOURCES(ds_0,ds_1,ds_2),
+TYPE(NAME='MySQL.MGR',
PROPERTIES('group-name'='92504d5b-6dec-11e8-91ea-246e9612aaf1')),
+HEARTBEAT(PROPERTIES('keep-alive-cron'='0/5 * * * * ?'))
+);
