This is an automated email from the ASF dual-hosted git repository.
jianglongtao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new f068bb1e057 Add EXPORT STORAGE NODES, refactor EXPORT/IMPORT METADATA
(#24488)
f068bb1e057 is described below
commit f068bb1e057c8b52640b99a3842c6ada6f1c64e0
Author: ChenJiaHao <[email protected]>
AuthorDate: Wed Mar 8 14:52:04 2023 +0800
Add EXPORT STORAGE NODES, refactor EXPORT/IMPORT METADATA (#24488)
* Add EXPORT STORAGE NODES, refactor EXPORT/IMPORT METADATA
* Add test cases
* Fix IMPORT METADATA test cases
* Fix comments
* Add blank line
---
.../parser/src/main/antlr4/imports/RALStatement.g4 | 4 +
.../parser/autogen/KernelDistSQLStatement.g4 | 1 +
.../core/kernel/KernelDistSQLStatementVisitor.java | 7 ++
.../ral/queryable/ExportStorageNodesStatement.java | 31 ++++----
.../distsql/export/ExportedClusterInfo.java | 9 ++-
.../backend/distsql/export/ExportedMetaData.java | 3 -
...dClusterInfo.java => ExportedSnapshotInfo.java} | 12 +--
...dClusterInfo.java => ExportedStorageNodes.java} | 8 +-
.../ral/queryable/ExportMetaDataExecutor.java | 39 ++--------
...ecutor.java => ExportStorageNodesExecutor.java} | 70 +++--------------
.../ral/updatable/ImportMetaDataUpdater.java | 1 +
....distsql.handler.ral.query.QueryableRALExecutor | 1 +
.../ral/queryable/ExportMetaDataExecutorTest.java | 6 +-
...st.java => ExportStorageNodesExecutorTest.java} | 90 ++++++++++++----------
.../ral/updatable/ImportMetaDataUpdaterTest.java | 4 +-
.../test/resources/conf/import/empty-metadata.json | 2 +-
.../expected/export-metadata-configuration.json | 2 +-
.../resources/expected/export-storage-nodes.json | 1 +
18 files changed, 121 insertions(+), 170 deletions(-)
diff --git a/distsql/parser/src/main/antlr4/imports/RALStatement.g4
b/distsql/parser/src/main/antlr4/imports/RALStatement.g4
index fba79333737..64a0408ec0c 100644
--- a/distsql/parser/src/main/antlr4/imports/RALStatement.g4
+++ b/distsql/parser/src/main/antlr4/imports/RALStatement.g4
@@ -91,6 +91,10 @@ importMetaData
: IMPORT METADATA (metaDataValue | FROM FILE filePath)
;
+exportStorageNodes
+ : EXPORT STORAGE NODES (TO FILE filePath)?
+ ;
+
convertYamlConfiguration
: CONVERT YAML CONFIGURATION FROM FILE filePath
;
diff --git
a/distsql/parser/src/main/antlr4/org/apache/shardingsphere/distsql/parser/autogen/KernelDistSQLStatement.g4
b/distsql/parser/src/main/antlr4/org/apache/shardingsphere/distsql/parser/autogen/KernelDistSQLStatement.g4
index 842ab54e121..daf6ce5af40 100644
---
a/distsql/parser/src/main/antlr4/org/apache/shardingsphere/distsql/parser/autogen/KernelDistSQLStatement.g4
+++
b/distsql/parser/src/main/antlr4/org/apache/shardingsphere/distsql/parser/autogen/KernelDistSQLStatement.g4
@@ -48,6 +48,7 @@ execute
| importDatabaseConfiguration
| exportMetaData
| importMetaData
+ | exportStorageNodes
| convertYamlConfiguration
| showMigrationRule
| alterMigrationRule
diff --git
a/distsql/parser/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java
b/distsql/parser/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java
index 8415701d7fa..47cd7332329 100644
---
a/distsql/parser/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java
+++
b/distsql/parser/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java
@@ -31,6 +31,7 @@ import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementPa
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.EnableComputeNodeContext;
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.ExportDatabaseConfigurationContext;
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.ExportMetaDataContext;
+import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.ExportStorageNodesContext;
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.FromSegmentContext;
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.ImportDatabaseConfigurationContext;
import
org.apache.shardingsphere.distsql.parser.autogen.KernelDistSQLStatementParser.ImportMetaDataContext;
@@ -77,6 +78,7 @@ import
org.apache.shardingsphere.distsql.parser.segment.URLBasedDataSourceSegmen
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ConvertYamlConfigurationStatement;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportDatabaseConfigurationStatement;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportMetaDataStatement;
+import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportStorageNodesStatement;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ShowComputeNodeInfoStatement;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ShowComputeNodeModeStatement;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ShowComputeNodesStatement;
@@ -300,6 +302,11 @@ public final class KernelDistSQLStatementVisitor extends
KernelDistSQLStatementB
return new ExportMetaDataStatement(null == ctx.filePath() ? null :
getIdentifierValue(ctx.filePath()));
}
+ @Override
+ public ASTNode visitExportStorageNodes(final ExportStorageNodesContext
ctx) {
+ return new ExportStorageNodesStatement(null == ctx.filePath() ? null :
getIdentifierValue(ctx.filePath()));
+ }
+
@Override
public ASTNode visitConvertYamlConfiguration(final
ConvertYamlConfigurationContext ctx) {
return new
ConvertYamlConfigurationStatement(getIdentifierValue(ctx.filePath()));
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
b/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/parser/statement/ral/queryable/ExportStorageNodesStatement.java
similarity index 59%
copy from
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
copy to
distsql/statement/src/main/java/org/apache/shardingsphere/distsql/parser/statement/ral/queryable/ExportStorageNodesStatement.java
index 49046410ba7..2e3af2638b1 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
+++
b/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/parser/statement/ral/queryable/ExportStorageNodesStatement.java
@@ -15,26 +15,27 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.proxy.backend.distsql.export;
+package org.apache.shardingsphere.distsql.parser.statement.ral.queryable;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.RequiredArgsConstructor;
+import
org.apache.shardingsphere.distsql.parser.statement.ral.QueryableRALStatement;
-import java.util.Collection;
-import java.util.Map;
+import java.util.Optional;
/**
- * Exported meta data.
+ * Export storage nodes statement.
*/
-@Getter
-@Setter
-public class ExportedMetaData {
+@RequiredArgsConstructor
+public final class ExportStorageNodesStatement extends QueryableRALStatement {
- private Collection<ExportedStorageNode> storageNodes;
+ private final String filePath;
- private Map<String, String> databases;
-
- private String props;
-
- private String rules;
+ /**
+ * Get file path.
+ *
+ * @return file path
+ */
+ public Optional<String> getFilePath() {
+ return Optional.ofNullable(filePath);
+ }
}
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
index 52013d6e09c..3b0e761aa8d 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
@@ -17,11 +17,10 @@
package org.apache.shardingsphere.proxy.backend.distsql.export;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
-import java.util.Collection;
-
/**
* Exported cluster info.
*/
@@ -29,7 +28,9 @@ import java.util.Collection;
@Setter
public class ExportedClusterInfo {
- private Collection<ExportedStorageNode> storageNodes;
-
+ @JsonProperty("meta_data")
private ExportedMetaData metaData;
+
+ @JsonProperty("snapshot_info")
+ private ExportedSnapshotInfo snapshotInfo;
}
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
index 49046410ba7..992456c11f1 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedMetaData.java
@@ -20,7 +20,6 @@ package
org.apache.shardingsphere.proxy.backend.distsql.export;
import lombok.Getter;
import lombok.Setter;
-import java.util.Collection;
import java.util.Map;
/**
@@ -30,8 +29,6 @@ import java.util.Map;
@Setter
public class ExportedMetaData {
- private Collection<ExportedStorageNode> storageNodes;
-
private Map<String, String> databases;
private String props;
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedSnapshotInfo.java
similarity index 79%
copy from
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
copy to
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedSnapshotInfo.java
index 52013d6e09c..2c258da4495 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedSnapshotInfo.java
@@ -17,19 +17,21 @@
package org.apache.shardingsphere.proxy.backend.distsql.export;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
-import java.util.Collection;
+import java.time.LocalDateTime;
/**
- * Exported cluster info.
+ * Exported snapshot info.
*/
@Getter
@Setter
-public class ExportedClusterInfo {
+public class ExportedSnapshotInfo {
- private Collection<ExportedStorageNode> storageNodes;
+ private String csn;
- private ExportedMetaData metaData;
+ @JsonProperty("create_time")
+ private LocalDateTime createTime;
}
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedStorageNodes.java
similarity index 87%
copy from
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
copy to
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedStorageNodes.java
index 52013d6e09c..d9e2dff3223 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedClusterInfo.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/distsql/export/ExportedStorageNodes.java
@@ -17,19 +17,19 @@
package org.apache.shardingsphere.proxy.backend.distsql.export;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
import java.util.Collection;
/**
- * Exported cluster info.
+ * Exported storage nodes.
*/
@Getter
@Setter
-public class ExportedClusterInfo {
+public class ExportedStorageNodes {
+ @JsonProperty("storage_nodes")
private Collection<ExportedStorageNode> storageNodes;
-
- private ExportedMetaData metaData;
}
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
index cd7150eeda9..049fd4fc69a 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
@@ -17,12 +17,9 @@
package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable;
-import
org.apache.shardingsphere.distsql.handler.ral.query.DatabaseRequiredQueryableRALExecutor;
+import
org.apache.shardingsphere.distsql.handler.ral.query.MetaDataRequiredQueryableRALExecutor;
import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportMetaDataStatement;
import org.apache.shardingsphere.infra.config.rule.RuleConfiguration;
-import org.apache.shardingsphere.infra.database.metadata.DataSourceMetaData;
-import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
-import
org.apache.shardingsphere.infra.datasource.props.DataSourcePropertiesCreator;
import
org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
import
org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
@@ -32,7 +29,6 @@ import
org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigur
import org.apache.shardingsphere.proxy.backend.context.ProxyContext;
import
org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo;
import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData;
-import
org.apache.shardingsphere.proxy.backend.distsql.export.ExportedStorageNode;
import org.apache.shardingsphere.proxy.backend.util.ExportUtils;
import org.apache.shardingsphere.proxy.backend.util.JsonUtils;
@@ -48,7 +44,7 @@ import java.util.Properties;
/**
* Export metadata executor.
*/
-public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRALExecutor<ExportMetaDataStatement> {
+public final class ExportMetaDataExecutor implements
MetaDataRequiredQueryableRALExecutor<ExportMetaDataStatement> {
@Override
public Collection<String> getColumnNames() {
@@ -56,8 +52,8 @@ public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRA
}
@Override
- public Collection<LocalDataQueryResultRow> getRows(final
ShardingSphereDatabase database, final ExportMetaDataStatement sqlStatement) {
- String exportedData = generateExportData(database);
+ public Collection<LocalDataQueryResultRow> getRows(final
ShardingSphereMetaData metaData, final ExportMetaDataStatement sqlStatement) {
+ String exportedData = generateExportData(metaData);
if (sqlStatement.getFilePath().isPresent()) {
String filePath = sqlStatement.getFilePath().get();
ExportUtils.exportToFile(filePath, exportedData);
@@ -68,15 +64,13 @@ public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRA
new
LocalDataQueryResultRow(ProxyContext.getInstance().getContextManager().getInstanceContext().getInstance().getCurrentInstanceId(),
LocalDateTime.now(), exportedData));
}
- private String generateExportData(final ShardingSphereDatabase database) {
+ private String generateExportData(final ShardingSphereMetaData metaData) {
ProxyContext proxyContext = ProxyContext.getInstance();
- ShardingSphereMetaData metaData =
proxyContext.getContextManager().getMetaDataContexts().getMetaData();
- ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo();
-
exportedClusterInfo.setStorageNodes(generateExportStorageNodeData(database));
ExportedMetaData exportedMetaData = new ExportedMetaData();
exportedMetaData.setDatabases(getDatabases(proxyContext));
exportedMetaData.setProps(generatePropsData(metaData.getProps().getProps()));
exportedMetaData.setRules(generateRulesData(metaData.getGlobalRuleMetaData().getConfigurations()));
+ ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo();
exportedClusterInfo.setMetaData(exportedMetaData);
return JsonUtils.toJsonString(exportedClusterInfo);
}
@@ -93,27 +87,6 @@ public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRA
return result;
}
- private Collection<ExportedStorageNode>
generateExportStorageNodeData(final ShardingSphereDatabase database) {
- Map<String, ExportedStorageNode> storageNodes = new LinkedHashMap<>();
- database.getResourceMetaData().getDataSources().forEach((key, value)
-> {
- DataSourceMetaData dataSourceMetaData =
database.getResourceMetaData().getDataSourceMetaData(key);
- String databaseInstanceIp = dataSourceMetaData.getHostname() + ":"
+ dataSourceMetaData.getPort();
- if (storageNodes.containsKey(databaseInstanceIp)) {
- return;
- }
- ExportedStorageNode exportedStorageNode = new
ExportedStorageNode();
- exportedStorageNode.setIp(dataSourceMetaData.getHostname());
-
exportedStorageNode.setPort(String.valueOf(dataSourceMetaData.getPort()));
- exportedStorageNode.setDatabase(dataSourceMetaData.getCatalog());
- DataSourceProperties dataSourceProps =
DataSourcePropertiesCreator.create(value);
- Map<String, Object> standardProperties =
dataSourceProps.getConnectionPropertySynonyms().getStandardProperties();
-
exportedStorageNode.setUsername(String.valueOf(standardProperties.get("username")));
-
exportedStorageNode.setPassword(String.valueOf(standardProperties.get("password")));
- storageNodes.put(databaseInstanceIp, exportedStorageNode);
- });
- return storageNodes.values();
- }
-
private String generatePropsData(final Properties props) {
if (props.isEmpty()) {
return "";
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutor.java
similarity index 57%
copy from
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
copy to
proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutor.java
index cd7150eeda9..adaf8a9c490 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutor.java
@@ -18,21 +18,15 @@
package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable;
import
org.apache.shardingsphere.distsql.handler.ral.query.DatabaseRequiredQueryableRALExecutor;
-import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportMetaDataStatement;
-import org.apache.shardingsphere.infra.config.rule.RuleConfiguration;
+import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportStorageNodesStatement;
import org.apache.shardingsphere.infra.database.metadata.DataSourceMetaData;
import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
import
org.apache.shardingsphere.infra.datasource.props.DataSourcePropertiesCreator;
import
org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
-import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
import
org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
-import org.apache.shardingsphere.infra.util.spi.type.ordered.OrderedSPILoader;
-import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
-import
org.apache.shardingsphere.infra.yaml.config.swapper.rule.YamlRuleConfigurationSwapper;
import org.apache.shardingsphere.proxy.backend.context.ProxyContext;
-import
org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo;
-import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData;
import
org.apache.shardingsphere.proxy.backend.distsql.export.ExportedStorageNode;
+import
org.apache.shardingsphere.proxy.backend.distsql.export.ExportedStorageNodes;
import org.apache.shardingsphere.proxy.backend.util.ExportUtils;
import org.apache.shardingsphere.proxy.backend.util.JsonUtils;
@@ -42,21 +36,19 @@ import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
/**
- * Export metadata executor.
+ * Export storage nodes executor.
*/
-public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRALExecutor<ExportMetaDataStatement> {
+public final class ExportStorageNodesExecutor implements
DatabaseRequiredQueryableRALExecutor<ExportStorageNodesStatement> {
@Override
public Collection<String> getColumnNames() {
- return Arrays.asList("id", "create_time", "cluster_info");
+ return Arrays.asList("id", "create_time", "storage_nodes");
}
@Override
- public Collection<LocalDataQueryResultRow> getRows(final
ShardingSphereDatabase database, final ExportMetaDataStatement sqlStatement) {
+ public Collection<LocalDataQueryResultRow> getRows(final
ShardingSphereDatabase database, final ExportStorageNodesStatement
sqlStatement) {
String exportedData = generateExportData(database);
if (sqlStatement.getFilePath().isPresent()) {
String filePath = sqlStatement.getFilePath().get();
@@ -69,28 +61,9 @@ public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRA
}
private String generateExportData(final ShardingSphereDatabase database) {
- ProxyContext proxyContext = ProxyContext.getInstance();
- ShardingSphereMetaData metaData =
proxyContext.getContextManager().getMetaDataContexts().getMetaData();
- ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo();
-
exportedClusterInfo.setStorageNodes(generateExportStorageNodeData(database));
- ExportedMetaData exportedMetaData = new ExportedMetaData();
- exportedMetaData.setDatabases(getDatabases(proxyContext));
-
exportedMetaData.setProps(generatePropsData(metaData.getProps().getProps()));
-
exportedMetaData.setRules(generateRulesData(metaData.getGlobalRuleMetaData().getConfigurations()));
- exportedClusterInfo.setMetaData(exportedMetaData);
- return JsonUtils.toJsonString(exportedClusterInfo);
- }
-
- private Map<String, String> getDatabases(final ProxyContext proxyContext) {
- Map<String, String> result = new LinkedHashMap<>();
- proxyContext.getAllDatabaseNames().forEach(each -> {
- ShardingSphereDatabase database = proxyContext.getDatabase(each);
- if
(database.getResourceMetaData().getAllInstanceDataSourceNames().isEmpty()) {
- return;
- }
- result.put(each, ExportUtils.generateExportDatabaseData(database));
- });
- return result;
+ ExportedStorageNodes storageNodes = new ExportedStorageNodes();
+ storageNodes.setStorageNodes(generateExportStorageNodeData(database));
+ return JsonUtils.toJsonString(storageNodes);
}
private Collection<ExportedStorageNode>
generateExportStorageNodeData(final ShardingSphereDatabase database) {
@@ -114,31 +87,8 @@ public final class ExportMetaDataExecutor implements
DatabaseRequiredQueryableRA
return storageNodes.values();
}
- private String generatePropsData(final Properties props) {
- if (props.isEmpty()) {
- return "";
- }
- StringBuilder result = new StringBuilder();
- result.append("props:").append(System.lineSeparator());
- props.forEach((key, value) -> result.append("
").append(key).append(": ").append(value).append(System.lineSeparator()));
- return result.toString();
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private String generateRulesData(final Collection<RuleConfiguration>
rules) {
- if (rules.isEmpty()) {
- return "";
- }
- StringBuilder result = new StringBuilder();
- result.append("rules:").append(System.lineSeparator());
- for (Entry<RuleConfiguration, YamlRuleConfigurationSwapper> entry :
OrderedSPILoader.getServices(YamlRuleConfigurationSwapper.class,
rules).entrySet()) {
-
result.append(YamlEngine.marshal(Collections.singletonList(entry.getValue().swapToYamlConfiguration(entry.getKey()))));
- }
- return result.toString();
- }
-
@Override
public String getType() {
- return ExportMetaDataStatement.class.getName();
+ return ExportStorageNodesStatement.class.getName();
}
}
diff --git
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
index 9e8a2f7e24d..4a9519db635 100644
---
a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
+++
b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
@@ -65,6 +65,7 @@ public final class ImportMetaDataUpdater implements
RALUpdater<ImportMetaDataSta
ExportedMetaData exportedMetaData = exportedClusterInfo.getMetaData();
importServerConfig(exportedMetaData);
importDatabase(exportedMetaData);
+ // TODO restore snapshot info
}
private void importServerConfig(final ExportedMetaData exportedMetaData) {
diff --git
a/proxy/backend/core/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.ral.query.QueryableRALExecutor
b/proxy/backend/core/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.ral.query.QueryableRALExecutor
index a075df90bfc..c4f05f3f38e 100644
---
a/proxy/backend/core/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.ral.query.QueryableRALExecutor
+++
b/proxy/backend/core/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.ral.query.QueryableRALExecutor
@@ -20,6 +20,7 @@
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ShowComput
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ShowComputeNodeModeExecutor
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ExportDatabaseConfigurationExecutor
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ExportMetaDataExecutor
+org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ExportStorageNodesExecutor
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ConvertYamlConfigurationExecutor
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ShowDistVariableExecutor
org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable.ShowDistVariablesExecutor
diff --git
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
index 0797592030f..579b616892c 100644
---
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
+++
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
@@ -81,10 +81,10 @@ public final class ExportMetaDataExecutorTest {
when(database.getResourceMetaData().getDataSources()).thenReturn(Collections.emptyMap());
when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList());
ExportMetaDataStatement sqlStatement = new
ExportMetaDataStatement(null);
- Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(database, sqlStatement);
+ Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(contextManager.getMetaDataContexts().getMetaData(),
sqlStatement);
assertThat(actual.size(), is(1));
LocalDataQueryResultRow row = actual.iterator().next();
- assertThat(row.getCell(3),
is("{\"storageNodes\":[],\"metaData\":{\"databases\":{\"empty_metadata\":\"databaseName:
null\\ndataSources:\\nrules:\\n\"},\"props\":\"\",\"rules\":\"\"}}"));
+ assertThat(row.getCell(3),
is("{\"meta_data\":{\"databases\":{\"empty_metadata\":\"databaseName:
null\\ndataSources:\\nrules:\\n\"},\"props\":\"\",\"rules\":\"\"}}"));
}
private ContextManager mockEmptyContextManager() {
@@ -101,7 +101,7 @@ public final class ExportMetaDataExecutorTest {
when(database.getName()).thenReturn("normal_db");
ContextManager contextManager = mockContextManager();
when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager);
- Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(database, new ExportMetaDataStatement(null));
+ Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(contextManager.getMetaDataContexts().getMetaData(),
new ExportMetaDataStatement(null));
assertThat(actual.size(), is(1));
LocalDataQueryResultRow row = actual.iterator().next();
assertThat(row.getCell(3), is(loadExpectedRow()));
diff --git
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutorTest.java
similarity index 53%
copy from
proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
copy to
proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutorTest.java
index 0797592030f..221fdeec211 100644
---
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutorTest.java
+++
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportStorageNodesExecutorTest.java
@@ -18,19 +18,17 @@
package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable;
import lombok.SneakyThrows;
-import org.apache.shardingsphere.authority.rule.AuthorityRule;
-import
org.apache.shardingsphere.authority.rule.builder.DefaultAuthorityRuleConfigurationBuilder;
-import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportMetaDataStatement;
-import org.apache.shardingsphere.infra.config.props.ConfigurationProperties;
-import org.apache.shardingsphere.infra.config.props.ConfigurationPropertyKey;
+import
org.apache.shardingsphere.distsql.parser.statement.ral.queryable.ExportStorageNodesStatement;
+import org.apache.shardingsphere.infra.config.algorithm.AlgorithmConfiguration;
import
org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
-import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
import
org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
-import
org.apache.shardingsphere.infra.metadata.database.rule.ShardingSphereRuleMetaData;
-import org.apache.shardingsphere.mode.manager.ContextManager;
-import org.apache.shardingsphere.mode.metadata.MetaDataContexts;
-import org.apache.shardingsphere.mode.metadata.persist.MetaDataPersistService;
import org.apache.shardingsphere.proxy.backend.context.ProxyContext;
+import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration;
+import
org.apache.shardingsphere.sharding.api.config.rule.ShardingTableRuleConfiguration;
+import
org.apache.shardingsphere.sharding.api.config.strategy.keygen.KeyGenerateStrategyConfiguration;
+import
org.apache.shardingsphere.sharding.api.config.strategy.sharding.NoneShardingStrategyConfiguration;
+import
org.apache.shardingsphere.sharding.api.config.strategy.sharding.StandardShardingStrategyConfiguration;
+import org.apache.shardingsphere.test.fixture.jdbc.MockedDataSource;
import org.apache.shardingsphere.test.mock.AutoMockExtension;
import org.apache.shardingsphere.test.mock.StaticMockSettings;
import org.apache.shardingsphere.test.util.PropertiesBuilder;
@@ -38,14 +36,16 @@ import
org.apache.shardingsphere.test.util.PropertiesBuilder.Property;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import javax.sql.DataSource;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.Objects;
import java.util.Properties;
@@ -57,69 +57,81 @@ import static org.mockito.Mockito.when;
@ExtendWith(AutoMockExtension.class)
@StaticMockSettings(ProxyContext.class)
-public final class ExportMetaDataExecutorTest {
+public final class ExportStorageNodesExecutorTest {
private final ShardingSphereDatabase database =
mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS);
@Test
public void assertGetColumns() {
- Collection<String> columns = new
ExportMetaDataExecutor().getColumnNames();
+ Collection<String> columns = new
ExportStorageNodesExecutor().getColumnNames();
assertThat(columns.size(), is(3));
Iterator<String> columnIterator = columns.iterator();
assertThat(columnIterator.next(), is("id"));
assertThat(columnIterator.next(), is("create_time"));
- assertThat(columnIterator.next(), is("cluster_info"));
+ assertThat(columnIterator.next(), is("storage_nodes"));
}
@Test
public void assertExecuteWithEmptyMetaData() {
- ContextManager contextManager = mockEmptyContextManager();
-
when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager);
-
when(ProxyContext.getInstance().getAllDatabaseNames()).thenReturn(Collections.singleton("empty_metadata"));
when(database.getName()).thenReturn("empty_metadata");
when(database.getResourceMetaData().getAllInstanceDataSourceNames()).thenReturn(Collections.singleton("empty_metadata"));
when(database.getResourceMetaData().getDataSources()).thenReturn(Collections.emptyMap());
when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList());
- ExportMetaDataStatement sqlStatement = new
ExportMetaDataStatement(null);
- Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(database, sqlStatement);
+ ExportStorageNodesStatement sqlStatement = new
ExportStorageNodesStatement(null);
+ Collection<LocalDataQueryResultRow> actual = new
ExportStorageNodesExecutor().getRows(database, sqlStatement);
assertThat(actual.size(), is(1));
LocalDataQueryResultRow row = actual.iterator().next();
- assertThat(row.getCell(3),
is("{\"storageNodes\":[],\"metaData\":{\"databases\":{\"empty_metadata\":\"databaseName:
null\\ndataSources:\\nrules:\\n\"},\"props\":\"\",\"rules\":\"\"}}"));
- }
-
- private ContextManager mockEmptyContextManager() {
- ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS);
- MetaDataContexts metaDataContexts = new
MetaDataContexts(mock(MetaDataPersistService.class), new
ShardingSphereMetaData(new HashMap<>(),
- new ShardingSphereRuleMetaData(Collections.emptyList()),
- new ConfigurationProperties(new Properties())));
- when(result.getMetaDataContexts()).thenReturn(metaDataContexts);
- return result;
+ assertThat(row.getCell(3), is("{\"storage_nodes\":[]}"));
}
@Test
public void assertExecute() throws SQLException {
when(database.getName()).thenReturn("normal_db");
- ContextManager contextManager = mockContextManager();
-
when(ProxyContext.getInstance().getContextManager()).thenReturn(contextManager);
- Collection<LocalDataQueryResultRow> actual = new
ExportMetaDataExecutor().getRows(database, new ExportMetaDataStatement(null));
+
when(database.getResourceMetaData().getDataSources()).thenReturn(createDataSourceMap());
+
when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.singleton(createShardingRuleConfiguration()));
+ Collection<LocalDataQueryResultRow> actual = new
ExportStorageNodesExecutor().getRows(database, new
ExportStorageNodesStatement(null));
assertThat(actual.size(), is(1));
LocalDataQueryResultRow row = actual.iterator().next();
assertThat(row.getCell(3), is(loadExpectedRow()));
}
- private ContextManager mockContextManager() {
- ContextManager result = mock(ContextManager.class, RETURNS_DEEP_STUBS);
- MetaDataContexts metaDataContexts = new
MetaDataContexts(mock(MetaDataPersistService.class), new
ShardingSphereMetaData(Collections.singletonMap(database.getName(), database),
- new ShardingSphereRuleMetaData(Collections.singleton(new
AuthorityRule(new DefaultAuthorityRuleConfigurationBuilder().build(),
Collections.emptyMap()))),
- new ConfigurationProperties(PropertiesBuilder.build(new
Property(ConfigurationPropertyKey.SQL_SHOW.getKey(), "true")))));
- when(result.getMetaDataContexts()).thenReturn(metaDataContexts);
+ private Map<String, DataSource> createDataSourceMap() {
+ Map<String, DataSource> result = new LinkedHashMap<>(2, 1);
+ result.put("ds_0", createDataSource("demo_ds_0"));
+ result.put("ds_1", createDataSource("demo_ds_1"));
+ return result;
+ }
+
+ private DataSource createDataSource(final String name) {
+ MockedDataSource result = new MockedDataSource();
+ result.setUrl(String.format("jdbc:mock://127.0.0.1/%s", name));
+ result.setUsername("root");
+ result.setPassword("");
+ result.setMaxPoolSize(50);
+ result.setMinPoolSize(1);
+ return result;
+ }
+
+ private ShardingRuleConfiguration createShardingRuleConfiguration() {
+ ShardingRuleConfiguration result = new ShardingRuleConfiguration();
+ result.getTables().add(createTableRuleConfiguration());
+ result.setDefaultDatabaseShardingStrategy(new
StandardShardingStrategyConfiguration("order_id", "ds_inline"));
+ result.setDefaultTableShardingStrategy(new
NoneShardingStrategyConfiguration());
+ result.getKeyGenerators().put("snowflake", new
AlgorithmConfiguration("SNOWFLAKE", new Properties()));
+ result.getShardingAlgorithms().put("ds_inline", new
AlgorithmConfiguration("INLINE", PropertiesBuilder.build(new
Property("algorithm-expression", "ds_${order_id % 2}"))));
+ return result;
+ }
+
+ private ShardingTableRuleConfiguration createTableRuleConfiguration() {
+ ShardingTableRuleConfiguration result = new
ShardingTableRuleConfiguration("t_order", "ds_${0..1}.t_order_${0..1}");
+ result.setKeyGenerateStrategy(new
KeyGenerateStrategyConfiguration("order_id", "snowflake"));
return result;
}
@SneakyThrows(IOException.class)
private String loadExpectedRow() {
StringBuilder result = new StringBuilder();
- String fileName =
Objects.requireNonNull(ExportMetaDataExecutorTest.class.getResource("/expected/export-metadata-configuration.json")).getFile();
+ String fileName =
Objects.requireNonNull(ExportStorageNodesExecutorTest.class.getResource("/expected/export-storage-nodes.json")).getFile();
try (
FileReader fileReader = new FileReader(fileName);
BufferedReader reader = new BufferedReader(fileReader)) {
diff --git
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
index 7f3deaf034c..570e0050438 100644
---
a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
+++
b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
@@ -57,8 +57,8 @@ import static org.mockito.Mockito.when;
@MockitoSettings(strictness = Strictness.LENIENT)
public final class ImportMetaDataUpdaterTest {
- private static final String METADATA_VALUE =
"{\"storageNodes\":[{\"ip\":\"127.0.0.1\",\"port\":\"3306\",\"username\":\"root\",\"password\":\"\",\"database\":\"demo_ds_0\"}],"
- + "\"metaData\":{\"databases\":{\"sharding_db\":\"databaseName:
sharding_db\\ndataSources:\\nrules:\\n\"},\"props\":\"props:\\n
system-log-level: INFO\\n sql-show: false\\n\","
+ private static final String METADATA_VALUE =
"{\"meta_data\":{\"databases\":{\"sharding_db\":\"databaseName:
sharding_db\\ndataSources:\\nrules:\\n\"},"
+ + "\"props\":\"props:\\n system-log-level: INFO\\n sql-show:
false\\n\","
+ "\"rules\":\"rules:\\n- !AUTHORITY\\n privilege:\\n type:
ALL_PERMITTED\\n users:\\n - authenticationMethodName: ''\\n password:
root\\n user: root@%\\n\"}}";
private final String empty = "empty_metadata";
diff --git
a/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
b/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
index ce344551ba5..ede5beb0d9a 100644
--- a/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
+++ b/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
@@ -1 +1 @@
-{"storageNodes":[],"metaData":{"databases":{"empty_metadata":"databaseName:
empty_metadata\n"},"props":"","rules":""}}
\ No newline at end of file
+{"meta_data":{"databases":{"empty_metadata":"databaseName:
empty_metadata\n"},"props":"","rules":""}}
diff --git
a/proxy/backend/core/src/test/resources/expected/export-metadata-configuration.json
b/proxy/backend/core/src/test/resources/expected/export-metadata-configuration.json
index 3e3397c6ab3..e2f0bc96e35 100644
---
a/proxy/backend/core/src/test/resources/expected/export-metadata-configuration.json
+++
b/proxy/backend/core/src/test/resources/expected/export-metadata-configuration.json
@@ -1 +1 @@
-{"storageNodes":[],"metaData":{"databases":{},"props":"props:\n sql-show:
true\n","rules":"rules:\n- !AUTHORITY\n privilege:\n type: ALL_PERMITTED\n
users:\n - authenticationMethodName: ''\n password: root\n user:
root@%\n"}}
+{"meta_data":{"databases":{},"props":"props:\n sql-show:
true\n","rules":"rules:\n- !AUTHORITY\n privilege:\n type: ALL_PERMITTED\n
users:\n - authenticationMethodName: ''\n password: root\n user:
root@%\n"}}
diff --git
a/proxy/backend/core/src/test/resources/expected/export-storage-nodes.json
b/proxy/backend/core/src/test/resources/expected/export-storage-nodes.json
new file mode 100644
index 00000000000..d58b25ecc5d
--- /dev/null
+++ b/proxy/backend/core/src/test/resources/expected/export-storage-nodes.json
@@ -0,0 +1 @@
+{"storage_nodes":[{"port":"0","username":"root","password":""}]}
