This is an automated email from the ASF dual-hosted git repository.
menghaoran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 16c4b4d Optimize rule config process. (#7529)
16c4b4d is described below
commit 16c4b4d28dbaeb9d1d3c6d77b9d09884d940c091
Author: 邱鹿 Lucas <[email protected]>
AuthorDate: Mon Sep 21 18:03:32 2020 +0800
Optimize rule config process. (#7529)
* Optimize rule config process.
* Update usage document.
Co-authored-by: qiulu3 <Lucas209910>
---
.../user-manual/shardingsphere-scaling/usage.cn.md | 44 +++++++++++-----------
.../user-manual/shardingsphere-scaling/usage.en.md | 44 +++++++++++-----------
.../scaling/web/HttpServerHandlerTest.java | 19 +++-------
.../src/test/resources/config.json | 10 ++---
.../core/check/AbstractDataConsistencyChecker.java | 16 +-------
.../core/utils/ConfigurationYamlConverter.java | 10 ++++-
.../src/test/resources/config.json | 2 +-
7 files changed, 68 insertions(+), 77 deletions(-)
diff --git
a/docs/document/content/user-manual/shardingsphere-scaling/usage.cn.md
b/docs/document/content/user-manual/shardingsphere-scaling/usage.cn.md
index 8c16255..bbed79d 100644
--- a/docs/document/content/user-manual/shardingsphere-scaling/usage.cn.md
+++ b/docs/document/content/user-manual/shardingsphere-scaling/usage.cn.md
@@ -89,27 +89,29 @@ curl -X POST \
password: scaling
",
"sourceRule":"
- tables:
- t_order:
- actualDataNodes: ds_$->{0..1}.t_order_$->{0..1}
- databaseStrategy:
- standard:
- shardingColumn: order_id
- shardingAlgorithmName: t_order_db_algorith
- logicTable: t_order
- tableStrategy:
- standard:
- shardingColumn: user_id
- shardingAlgorithmName: t_order_tbl_algorith
- shardingAlgorithms:
- t_order_db_algorith:
- type: INLINE
- props:
- algorithm-expression: ds_$->{order_id % 2}
- t_order_tbl_algorith:
- type: INLINE
- props:
- algorithm-expression: t_order_$->{user_id % 2}
+ rules:
+ - !SHARDING
+ tables:
+ t_order:
+ actualDataNodes: ds_$->{0..1}.t_order_$->{0..1}
+ databaseStrategy:
+ standard:
+ shardingColumn: order_id
+ shardingAlgorithmName: t_order_db_algorith
+ logicTable: t_order
+ tableStrategy:
+ standard:
+ shardingColumn: user_id
+ shardingAlgorithmName: t_order_tbl_algorith
+ shardingAlgorithms:
+ t_order_db_algorith:
+ type: INLINE
+ props:
+ algorithm-expression: ds_$->{order_id % 2}
+ t_order_tbl_algorith:
+ type: INLINE
+ props:
+ algorithm-expression: t_order_$->{user_id % 2}
",
"destinationDataSources":{
"username":"root",
diff --git
a/docs/document/content/user-manual/shardingsphere-scaling/usage.en.md
b/docs/document/content/user-manual/shardingsphere-scaling/usage.en.md
index 16272ef..2004562 100644
--- a/docs/document/content/user-manual/shardingsphere-scaling/usage.en.md
+++ b/docs/document/content/user-manual/shardingsphere-scaling/usage.en.md
@@ -89,27 +89,29 @@ curl -X POST \
password: scaling
",
"sourceRule":"
- tables:
- t_order:
- actualDataNodes: ds_$->{0..1}.t_order_$->{0..1}
- databaseStrategy:
- standard:
- shardingColumn: order_id
- shardingAlgorithmName: t_order_db_algorith
- logicTable: t_order
- tableStrategy:
- standard:
- shardingColumn: user_id
- shardingAlgorithmName: t_order_tbl_algorith
- shardingAlgorithms:
- t_order_db_algorith:
- type: INLINE
- props:
- algorithm-expression: ds_$->{order_id % 2}
- t_order_tbl_algorith:
- type: INLINE
- props:
- algorithm-expression: t_order_$->{user_id % 2}
+ rules:
+ - !SHARDING
+ tables:
+ t_order:
+ actualDataNodes: ds_$->{0..1}.t_order_$->{0..1}
+ databaseStrategy:
+ standard:
+ shardingColumn: order_id
+ shardingAlgorithmName: t_order_db_algorith
+ logicTable: t_order
+ tableStrategy:
+ standard:
+ shardingColumn: user_id
+ shardingAlgorithmName: t_order_tbl_algorith
+ shardingAlgorithms:
+ t_order_db_algorith:
+ type: INLINE
+ props:
+ algorithm-expression: ds_$->{order_id % 2}
+ t_order_tbl_algorith:
+ type: INLINE
+ props:
+ algorithm-expression: t_order_$->{user_id % 2}
",
"destinationDataSources":{
"username":"root",
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/java/org/apache/shardingsphere/scaling/web/HttpServerHandlerTest.java
b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/java/org/apache/shardingsphere/scaling/web/HttpServerHandlerTest.java
index 7faf4c7..a0dd379 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/java/org/apache/shardingsphere/scaling/web/HttpServerHandlerTest.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/java/org/apache/shardingsphere/scaling/web/HttpServerHandlerTest.java
@@ -77,19 +77,6 @@ public final class HttpServerHandlerTest {
assertTrue(fullHttpResponse.content().toString(CharsetUtil.UTF_8).contains("{\"success\":true"));
}
- private void startScalingJob() {
-
scalingConfiguration.getRuleConfiguration().setSourceDatasource("dataSources:\n
ds_0:\n "
- + "dataSourceClassName: com.zaxxer.hikari.HikariDataSource\n
props:\n "
- + "jdbcUrl:
jdbc:h2:mem:test_db;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL\n
username: root\n password: 'password'\n connectionTimeout: 30000\n "
- + "idleTimeout: 60000\n maxLifetime: 1800000\n
maxPoolSize: 50\n minPoolSize: 1\n maintenanceIntervalMilliseconds:
30000\n readOnly: false\n");
-
scalingConfiguration.getRuleConfiguration().getDestinationDataSources().setUrl("jdbc:h2:mem:test_db;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL");
-
scalingConfiguration.getRuleConfiguration().getDestinationDataSources().setName("root");
-
scalingConfiguration.getRuleConfiguration().getDestinationDataSources().setPassword("password");
- ByteBuf byteBuf =
Unpooled.copiedBuffer(GSON.toJson(scalingConfiguration), CharsetUtil.UTF_8);
- fullHttpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1,
HttpMethod.POST, "/scaling/job/start", byteBuf);
- httpServerHandler.channelRead0(channelHandlerContext, fullHttpRequest);
- }
-
@Test
public void assertChannelReadProgressFail() {
fullHttpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1,
HttpMethod.GET, "/scaling/job/progress/9");
@@ -160,6 +147,12 @@ public final class HttpServerHandlerTest {
httpServerHandler.exceptionCaught(channelHandlerContext, throwable);
verify(channelHandlerContext).close();
}
+
+ private void startScalingJob() {
+ ByteBuf byteBuf =
Unpooled.copiedBuffer(GSON.toJson(scalingConfiguration), CharsetUtil.UTF_8);
+ fullHttpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1,
HttpMethod.POST, "/scaling/job/start", byteBuf);
+ httpServerHandler.channelRead0(channelHandlerContext, fullHttpRequest);
+ }
private void initConfig(final String configFile) {
InputStream fileInputStream =
HttpServerHandlerTest.class.getResourceAsStream(configFile);
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/resources/config.json
b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/resources/config.json
index bbd1c05..d85d156 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/resources/config.json
+++
b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/test/resources/config.json
@@ -17,13 +17,13 @@
{
"ruleConfiguration": {
- "sourceDatasource":"dataSources:\n ds_0:\n dataSourceClassName:
com.zaxxer.hikari.HikariDataSource\n props:\n jdbcUrl:
jdbc:mysql://127.0.0.1:3306/test?serverTimezone=UTC&useSSL=false\n username:
root\n password: '123456'\n connectionTimeout: 30000\n idleTimeout:
60000\n maxLifetime: 1800000\n maxPoolSize: 50\n minPoolSize: 1\n
maintenanceIntervalMilliseconds: 30000\n readOnly: false\n",
- "sourceRule": "defaultDatabaseStrategy:\n standard:\n
shardingAlgorithmName: inline\n shardingColumn: user_id\ntables:\n t1:\n
actualDataNodes: ds_0.t1\n keyGenerateStrategy:\n column: order_id\n
logicTable: t1\n tableStrategy:\n standard:\n
shardingAlgorithmName: inline\n shardingColumn: order_id\n t2:\n
actualDataNodes: ds_0.t2\n keyGenerateStrategy:\n column:
order_item_id\n logicTable: t2\n tableStrategy:\n standa [...]
+ "sourceDatasource":"dataSources:\n ds_0:\n dataSourceClassName:
com.zaxxer.hikari.HikariDataSource\n props:\n jdbcUrl:
jdbc:h2:mem:test_db_2;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL\n
username: root\n password: '123456'\n connectionTimeout: 30000\n
idleTimeout: 60000\n maxLifetime: 1800000\n maxPoolSize: 50\n
minPoolSize: 1\n maintenanceIntervalMilliseconds: 30000\n readOnly:
false\n",
+ "sourceRule": "rules:\n- !SHARDING\n defaultDatabaseStrategy:\n
standard:\n shardingAlgorithmName: inline\n shardingColumn: user_id\n
tables:\n t1:\n actualDataNodes: ds_0.t1\n keyGenerateStrategy:\n
column: order_id\n logicTable: t1\n tableStrategy:\n
standard:\n shardingAlgorithmName: inline\n shardingColumn:
order_id\n t2:\n actualDataNodes: ds_0.t2\n keyGenerateStrategy:\n
column: order_item [...]
"destinationDataSources": {
"name": "dt_0",
- "password": "123456",
- "url":
"jdbc:mysql://127.0.0.1:3306/test2?serverTimezone=UTC&useSSL=false",
- "username": "root"
+ "url":
"jdbc:h2:mem:test_db_2;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL",
+ "username": "root",
+ "password": "password"
}
},
"jobConfiguration": {
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/check/AbstractDataConsistencyChecker.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/check/AbstractDataConsistencyChecker.java
index 721a38f..f51e536 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/check/AbstractDataConsistencyChecker.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/check/AbstractDataConsistencyChecker.java
@@ -49,10 +49,6 @@ public abstract class AbstractDataConsistencyChecker
implements DataConsistencyC
private final ShardingScalingJob shardingScalingJob;
- private DataSource sourceDataSource;
-
- private DataSource destinationDataSource;
-
@Override
public boolean countCheck() {
return
shardingScalingJob.getSyncConfigurations().stream().allMatch(each ->
each.getDumperConfiguration().getTableNameMap().values().stream().distinct().allMatch(this::countCheck));
@@ -68,27 +64,19 @@ public abstract class AbstractDataConsistencyChecker
implements DataConsistencyC
}
protected DataSource getSourceDataSource() {
- if (sourceDataSource != null) {
- return sourceDataSource;
- }
try {
Map<String, DataSource> dataSourceMap =
DataSourceConverter.getDataSourceMap(
ConfigurationYamlConverter.loadDataSourceConfigurations(shardingScalingJob.getScalingConfiguration().getRuleConfiguration().getSourceDatasource()));
ShardingRuleConfiguration ruleConfiguration =
ConfigurationYamlConverter.loadShardingRuleConfiguration(shardingScalingJob.getScalingConfiguration().getRuleConfiguration().getSourceRule());
- sourceDataSource =
ShardingSphereDataSourceFactory.createDataSource(dataSourceMap,
Lists.newArrayList(ruleConfiguration), null);
+ return
ShardingSphereDataSourceFactory.createDataSource(dataSourceMap,
Lists.newArrayList(ruleConfiguration), null);
} catch (SQLException ex) {
throw new DataCheckFailException("get source data source failed.",
ex);
}
- return sourceDataSource;
}
protected DataSource getDestinationDataSource() {
- if (destinationDataSource != null) {
- return destinationDataSource;
- }
RuleConfiguration.YamlDataSourceParameter parameter =
shardingScalingJob.getScalingConfiguration().getRuleConfiguration().getDestinationDataSources();
- destinationDataSource = new DataSourceFactory().newInstance(new
JDBCDataSourceConfiguration(parameter.getUrl(), parameter.getUsername(),
parameter.getPassword()));
- return destinationDataSource;
+ return new DataSourceFactory().newInstance(new
JDBCDataSourceConfiguration(parameter.getUrl(), parameter.getUsername(),
parameter.getPassword()));
}
private long count(final DataSource dataSource, final String table) {
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/utils/ConfigurationYamlConverter.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/utils/ConfigurationYamlConverter.java
index 990eda7..250a537 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/utils/ConfigurationYamlConverter.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/utils/ConfigurationYamlConverter.java
@@ -22,14 +22,17 @@ import com.google.common.collect.Maps;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import
org.apache.shardingsphere.governance.core.yaml.config.YamlDataSourceConfigurationWrap;
+import
org.apache.shardingsphere.governance.core.yaml.swapper.DataSourceConfigurationYamlSwapper;
import
org.apache.shardingsphere.infra.config.datasource.DataSourceConfiguration;
+import org.apache.shardingsphere.infra.yaml.config.YamlRootRuleConfigurations;
+import org.apache.shardingsphere.infra.yaml.config.YamlRuleConfiguration;
import org.apache.shardingsphere.infra.yaml.engine.YamlEngine;
-import
org.apache.shardingsphere.governance.core.yaml.swapper.DataSourceConfigurationYamlSwapper;
import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration;
import
org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration;
import
org.apache.shardingsphere.sharding.yaml.swapper.ShardingRuleConfigurationYamlSwapper;
import java.util.Map;
+import java.util.Optional;
/**
* YAML converter for configuration.
@@ -56,6 +59,9 @@ public final class ConfigurationYamlConverter {
* @return sharding rule configuration
*/
public static ShardingRuleConfiguration
loadShardingRuleConfiguration(final String data) {
- return new
ShardingRuleConfigurationYamlSwapper().swapToObject(YamlEngine.unmarshal(data,
YamlShardingRuleConfiguration.class));
+ YamlRootRuleConfigurations rootRuleConfigurations =
YamlEngine.unmarshal(data, YamlRootRuleConfigurations.class);
+ Optional<YamlRuleConfiguration> ruleConfiguration =
rootRuleConfigurations.getRules().stream().filter(each -> each instanceof
YamlShardingRuleConfiguration).findFirst();
+ Preconditions.checkState(ruleConfiguration.isPresent(), "No available
sharding rule to load for governance.");
+ return new
ShardingRuleConfigurationYamlSwapper().swapToObject((YamlShardingRuleConfiguration)
ruleConfiguration.get());
}
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/config.json
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/config.json
index cf1c407..34b11de 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/config.json
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/config.json
@@ -18,7 +18,7 @@
{
"ruleConfiguration": {
"sourceDatasource": "dataSources:\n ds_0:\n dataSourceClassName:
com.zaxxer.hikari.HikariDataSource\n props:\n jdbcUrl:
jdbc:h2:mem:test_db_1;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL\n
username: root\n password: 'password'\n connectionTimeout: 30000\n
idleTimeout: 60000\n maxLifetime: 1800000\n maxPoolSize: 50\n
minPoolSize: 1\n maintenanceIntervalMilliseconds: 30000\n readOnly:
false\n",
- "sourceRule": "defaultDatabaseStrategy:\n standard:\n
shardingAlgorithmName: inline\n shardingColumn: user_id\ntables:\n t1:\n
actualDataNodes: ds_0.t1\n keyGenerateStrategy:\n column: order_id\n
logicTable: t1\n tableStrategy:\n standard:\n
shardingAlgorithmName: inline\n shardingColumn: order_id\n t2:\n
actualDataNodes: ds_0.t2\n keyGenerateStrategy:\n column:
order_item_id\n logicTable: t2\n tableStrategy:\n standa [...]
+ "sourceRule": "rules:\n- !SHARDING\n defaultDatabaseStrategy:\n
standard:\n shardingAlgorithmName: inline\n shardingColumn: user_id\n
tables:\n t1:\n actualDataNodes: ds_0.t1\n keyGenerateStrategy:\n
column: order_id\n logicTable: t1\n tableStrategy:\n
standard:\n shardingAlgorithmName: inline\n shardingColumn:
order_id\n t2:\n actualDataNodes: ds_0.t2\n keyGenerateStrategy:\n
column: order_item [...]
"destinationDataSources": {
"name": "dt_0",
"url":
"jdbc:h2:mem:test_db_2;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=false;MODE=MySQL",
