This is an automated email from the ASF dual-hosted git repository. jiangmaolin pushed a commit to branch dev-5.5.1 in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
commit 1acfd04598f3e254eecbc1033c610ec603e77687 Author: jiangML <[email protected]> AuthorDate: Tue Nov 12 21:08:12 2024 +0800 Improve export database config to json type --- .../engine/src/main/antlr4/imports/RALStatement.g4 | 12 ++- .../core/kernel/KernelDistSQLStatementVisitor.java | 9 +- .../ExportDatabaseConfigurationStatement.java | 14 +++ .../ImportDatabaseConfigurationStatement.java | 16 ++++ .../ExportDatabaseConfigurationExecutor.java | 41 ++++++++- .../ImportDatabaseConfigurationExecutor.java | 99 ++++++++++++++++++++-- .../ExportDatabaseConfigurationExecutorTest.java | 8 +- .../ImportDatabaseConfigurationExecutorTest.java | 4 +- 8 files changed, 184 insertions(+), 19 deletions(-) diff --git a/parser/distsql/engine/src/main/antlr4/imports/RALStatement.g4 b/parser/distsql/engine/src/main/antlr4/imports/RALStatement.g4 index eb38ddcdfb0..8aeffdf51fb 100644 --- a/parser/distsql/engine/src/main/antlr4/imports/RALStatement.g4 +++ b/parser/distsql/engine/src/main/antlr4/imports/RALStatement.g4 @@ -78,11 +78,15 @@ unlabelComputeNode ; exportDatabaseConfiguration - : EXPORT DATABASE CONFIGURATION (FROM databaseName)? (TO FILE filePath)? + // SPEX CHANGED: BEGIN + : EXPORT DATABASE CONFIGURATION (FROM databaseName)? (TO FILE filePath)? (COMMA_ TYPE EQ_ exportContentType)? + // SPEX CHANGED: END ; importDatabaseConfiguration - : IMPORT DATABASE CONFIGURATION FROM FILE filePath + // SPEX CHANGED: BEGIN + : IMPORT DATABASE CONFIGURATION (metaDataValue | FROM FILE filePath) + // SPEX CHANGED: END ; exportMetaData @@ -176,3 +180,7 @@ likePattern pluginClass : STRING_ ; + +exportContentType + : STRING_ + ; diff --git a/parser/distsql/engine/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java b/parser/distsql/engine/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java index eeafafe87a7..909dd1d1b64 100644 --- a/parser/distsql/engine/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java +++ b/parser/distsql/engine/src/main/java/org/apache/shardingsphere/distsql/parser/core/kernel/KernelDistSQLStatementVisitor.java @@ -299,7 +299,10 @@ public final class KernelDistSQLStatementVisitor extends KernelDistSQLStatementB @Override public ASTNode visitExportDatabaseConfiguration(final ExportDatabaseConfigurationContext ctx) { - return new ExportDatabaseConfigurationStatement(null == ctx.databaseName() ? null : (DatabaseSegment) visit(ctx.databaseName()), getIdentifierValue(ctx.filePath())); + // SPEX CHANGED: BEGIN + return new ExportDatabaseConfigurationStatement(null == ctx.databaseName() ? null : (DatabaseSegment) visit(ctx.databaseName()), + getIdentifierValue(ctx.filePath()), getIdentifierValue(ctx.exportContentType())); + // SPEX CHANGED: END } @Override @@ -324,12 +327,14 @@ public final class KernelDistSQLStatementVisitor extends KernelDistSQLStatementB @Override public ASTNode visitImportDatabaseConfiguration(final ImportDatabaseConfigurationContext ctx) { - return new ImportDatabaseConfigurationStatement(getIdentifierValue(ctx.filePath())); + return new ImportDatabaseConfigurationStatement(getIdentifierValue(ctx.metaDataValue()), getIdentifierValue(ctx.filePath())); } @Override public ASTNode visitImportMetaData(final ImportMetaDataContext ctx) { + // SPEX CHANGED: BEGIN return new ImportMetaDataStatement(null == ctx.metaDataValue() ? null : getQuotedContent(ctx.metaDataValue()), getIdentifierValue(ctx.filePath())); + // SPEX CHANGED: END } @Override diff --git a/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/queryable/export/ExportDatabaseConfigurationStatement.java b/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/queryable/export/ExportDatabaseConfigurationStatement.java index bc5f0e116bb..2cab1731f41 100644 --- a/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/queryable/export/ExportDatabaseConfigurationStatement.java +++ b/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/queryable/export/ExportDatabaseConfigurationStatement.java @@ -17,6 +17,7 @@ package org.apache.shardingsphere.distsql.statement.ral.queryable.export; +import com.sphereex.dbplusengine.SphereEx; import lombok.RequiredArgsConstructor; import org.apache.shardingsphere.distsql.statement.ral.queryable.QueryableRALStatement; import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.DatabaseSegment; @@ -34,6 +35,9 @@ public final class ExportDatabaseConfigurationStatement extends QueryableRALStat private final String filePath; + @SphereEx + private final String exportContentType; + @Override public Optional<DatabaseSegment> getDatabase() { return Optional.ofNullable(database); @@ -47,4 +51,14 @@ public final class ExportDatabaseConfigurationStatement extends QueryableRALStat public Optional<String> getFilePath() { return Optional.ofNullable(filePath); } + + /** + * Get export content type. + * + * @return type + */ + @SphereEx + public Optional<String> getExportContentType() { + return Optional.ofNullable(exportContentType); + } } diff --git a/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/updatable/ImportDatabaseConfigurationStatement.java b/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/updatable/ImportDatabaseConfigurationStatement.java index 94f19725942..c284e466200 100644 --- a/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/updatable/ImportDatabaseConfigurationStatement.java +++ b/parser/distsql/statement/src/main/java/org/apache/shardingsphere/distsql/statement/ral/updatable/ImportDatabaseConfigurationStatement.java @@ -17,9 +17,12 @@ package org.apache.shardingsphere.distsql.statement.ral.updatable; +import com.sphereex.dbplusengine.SphereEx; import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.util.Optional; + /** * Import database configuration statement. */ @@ -27,5 +30,18 @@ import lombok.RequiredArgsConstructor; @Getter public final class ImportDatabaseConfigurationStatement extends UpdatableRALStatement { + @SphereEx + private final String metaDataValue; + private final String filePath; + + /** + * Get file path. + * + * @return file path + */ + @SphereEx + public Optional<String> getFilePath() { + return Optional.ofNullable(filePath); + } } diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java index b95585bb757..0e4e676aa81 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutor.java @@ -17,13 +17,21 @@ package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable; +import com.sphereex.dbplusengine.SphereEx; +import com.sphereex.dbplusengine.SphereEx.Type; import lombok.Setter; +import org.apache.commons.codec.binary.Base64; import org.apache.shardingsphere.distsql.handler.aware.DistSQLExecutorDatabaseAware; import org.apache.shardingsphere.distsql.handler.engine.query.DistSQLQueryExecutor; import org.apache.shardingsphere.distsql.statement.ral.queryable.export.ExportDatabaseConfigurationStatement; +import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; +import org.apache.shardingsphere.infra.exception.dialect.exception.data.InvalidParameterValueException; import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow; import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase; +import org.apache.shardingsphere.infra.util.json.JsonUtils; import org.apache.shardingsphere.mode.manager.ContextManager; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; import org.apache.shardingsphere.proxy.backend.util.ExportUtils; import java.util.Collection; @@ -42,17 +50,42 @@ public final class ExportDatabaseConfigurationExecutor implements DistSQLQueryEx return Collections.singleton("result"); } + @SphereEx(Type.MODIFY) @Override public Collection<LocalDataQueryResultRow> getRows(final ExportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) { String exportedData = ExportUtils.generateExportDatabaseData(database); - if (!sqlStatement.getFilePath().isPresent()) { - return Collections.singleton(new LocalDataQueryResultRow(exportedData)); - } + return sqlStatement.getFilePath().isPresent() ? exportToFile(sqlStatement, exportedData) : exportToConsole(sqlStatement, exportedData); + } + + @SphereEx + private Collection<LocalDataQueryResultRow> exportToFile(final ExportDatabaseConfigurationStatement sqlStatement, final String exportedData) { String filePath = sqlStatement.getFilePath().get(); - ExportUtils.exportToFile(filePath, exportedData); + if (!sqlStatement.getExportContentType().isPresent()) { + ExportUtils.exportToFile(filePath, exportedData); + return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath))); + } + ShardingSpherePreconditions.checkState("json".equalsIgnoreCase(sqlStatement.getExportContentType().get()), () -> new InvalidParameterValueException("type", sqlStatement.getExportContentType().get())); + ExportUtils.exportToFile(filePath, buildJsonContent(exportedData)); return Collections.singleton(new LocalDataQueryResultRow(String.format("Successfully exported to: '%s'", filePath))); } + @SphereEx + private Collection<LocalDataQueryResultRow> exportToConsole(final ExportDatabaseConfigurationStatement sqlStatement, final String exportedData) { + if (!sqlStatement.getExportContentType().isPresent()) { + return Collections.singleton(new LocalDataQueryResultRow(exportedData)); + } + ShardingSpherePreconditions.checkState("json".equalsIgnoreCase(sqlStatement.getExportContentType().get()), () -> new InvalidParameterValueException("type", sqlStatement.getExportContentType().get())); + return Collections.singleton(new LocalDataQueryResultRow(Base64.encodeBase64String(buildJsonContent(exportedData).getBytes()))); + } + + private String buildJsonContent(final String exportedData) { + ExportedMetaData exportedMetaData = new ExportedMetaData(); + exportedMetaData.setDatabases(Collections.singletonMap(database.getName(), exportedData)); + ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo(); + exportedClusterInfo.setMetaData(exportedMetaData); + return JsonUtils.toJsonString(exportedClusterInfo); + } + @Override public void setDatabase(final ShardingSphereDatabase database) { this.database = database; diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java index ec256317215..24d682cd786 100644 --- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java +++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutor.java @@ -17,17 +17,33 @@ package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.updatable; +import com.google.common.base.Strings; +import com.sphereex.dbplusengine.SphereEx; +import com.sphereex.dbplusengine.SphereEx.Type; +import com.sphereex.dbplusengine.infra.exception.metadata.EnableMetaDataConsistencyFailedException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.FileUtils; import org.apache.shardingsphere.distsql.handler.engine.update.DistSQLUpdateExecutor; import org.apache.shardingsphere.distsql.statement.ral.updatable.ImportDatabaseConfigurationStatement; +import org.apache.shardingsphere.infra.config.props.temporary.TemporaryConfigurationPropertyKey; +import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions; +import org.apache.shardingsphere.infra.exception.dialect.exception.data.InvalidParameterValueException; +import org.apache.shardingsphere.infra.util.json.JsonUtils; import org.apache.shardingsphere.infra.util.yaml.YamlEngine; import org.apache.shardingsphere.mode.manager.ContextManager; import org.apache.shardingsphere.proxy.backend.config.yaml.YamlProxyDatabaseConfiguration; -import org.apache.shardingsphere.infra.exception.generic.FileIOException; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedClusterInfo; +import org.apache.shardingsphere.proxy.backend.distsql.export.ExportedMetaData; import org.apache.shardingsphere.proxy.backend.util.YamlDatabaseConfigurationImportExecutor; import java.io.File; -import java.io.IOException; +import java.nio.charset.Charset; import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * Import database configuration executor. @@ -36,16 +52,83 @@ public final class ImportDatabaseConfigurationExecutor implements DistSQLUpdateE private final YamlDatabaseConfigurationImportExecutor databaseConfigImportExecutor = new YamlDatabaseConfigurationImportExecutor(); + @SphereEx(Type.MODIFY) @Override public void executeUpdate(final ImportDatabaseConfigurationStatement sqlStatement, final ContextManager contextManager) throws SQLException { - File file = new File(sqlStatement.getFilePath()); - YamlProxyDatabaseConfiguration yamlConfig; + if (sqlStatement.getFilePath().isPresent()) { + importFromFile(contextManager, sqlStatement); + return; + } + importFromConsole(contextManager, sqlStatement); + } + + @SphereEx + private void importFromFile(final ContextManager contextManager, final ImportDatabaseConfigurationStatement sqlStatement) throws SQLException { + File file = new File(sqlStatement.getFilePath().get()); + try { + String jsonMetaDataConfig = FileUtils.readFileToString(file, Charset.defaultCharset()); + ShardingSpherePreconditions.checkState(!Strings.isNullOrEmpty(jsonMetaDataConfig), () -> new InvalidParameterValueException("File", sqlStatement.getFilePath().get())); + ExportedClusterInfo exportedClusterInfo = JsonUtils.fromJsonString(jsonMetaDataConfig, ExportedClusterInfo.class); + ExportedMetaData exportedMetaData = exportedClusterInfo.getMetaData(); + enableMetaDataConsistency(contextManager); + importDatabase(exportedMetaData); + return; + } catch (final Exception ignored) { + } try { - yamlConfig = YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class); - } catch (final IOException ignore) { - throw new FileIOException(file); + YamlProxyDatabaseConfiguration yamlConfig = YamlEngine.unmarshal(file, YamlProxyDatabaseConfiguration.class); + enableMetaDataConsistency(contextManager); + databaseConfigImportExecutor.importDatabaseConfiguration(yamlConfig); + return; + } catch (final Exception ignored) { + } + throw new InvalidParameterValueException("File", sqlStatement.getFilePath().get()); + } + + @SphereEx + private void importFromConsole(final ContextManager contextManager, final ImportDatabaseConfigurationStatement sqlStatement) throws SQLException { + String jsonMetaDataConfig = new String(Base64.decodeBase64(sqlStatement.getMetaDataValue())); + ExportedClusterInfo exportedClusterInfo = JsonUtils.fromJsonString(jsonMetaDataConfig, ExportedClusterInfo.class); + ExportedMetaData exportedMetaData = exportedClusterInfo.getMetaData(); + enableMetaDataConsistency(contextManager); + importDatabase(exportedMetaData); + } + + @SphereEx + private void enableMetaDataConsistency(final ContextManager contextManager) throws SQLException { + if (contextManager.getMetaDataContexts().getMetaData().getTemporaryProps().<Boolean>getValue(TemporaryConfigurationPropertyKey.META_DATA_CONSISTENCY_ENABLED)) { + return; + } + contextManager.getPersistServiceFacade().getMetaDataManagerPersistService().alterProperties(getEnableMetaDataConsistencyProperties(contextManager)); + try { + CompletableFuture.runAsync(() -> { + while (!contextManager.getMetaDataContexts().getMetaData().getTemporaryProps().<Boolean>getValue(TemporaryConfigurationPropertyKey.META_DATA_CONSISTENCY_ENABLED)) { + try { + TimeUnit.MILLISECONDS.sleep(1000L); + } catch (final InterruptedException ignore) { + } + } + }).get(10L, TimeUnit.SECONDS); + } catch (final InterruptedException | ExecutionException | TimeoutException ignore) { + throw new EnableMetaDataConsistencyFailedException(); + } + } + + @SphereEx + private Properties getEnableMetaDataConsistencyProperties(final ContextManager contextManager) { + Properties result = new Properties(); + result.putAll(contextManager.getMetaDataContexts().getMetaData().getProps().getProps()); + result.putAll(contextManager.getMetaDataContexts().getMetaData().getTemporaryProps().getProps()); + result.put(TemporaryConfigurationPropertyKey.META_DATA_CONSISTENCY_ENABLED.getKey(), Boolean.TRUE.toString()); + return result; + } + + @SphereEx + private void importDatabase(final ExportedMetaData exportedMetaData) throws SQLException { + for (String each : exportedMetaData.getDatabases().values()) { + YamlProxyDatabaseConfiguration yamlDatabaseConfig = YamlEngine.unmarshal(each, YamlProxyDatabaseConfiguration.class); + databaseConfigImportExecutor.importDatabaseConfiguration(yamlDatabaseConfig); } - databaseConfigImportExecutor.importDatabaseConfiguration(yamlConfig); } @Override diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutorTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutorTest.java index 1b0de173e3d..52743f14c51 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutorTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportDatabaseConfigurationExecutorTest.java @@ -17,6 +17,8 @@ package org.apache.shardingsphere.proxy.backend.handler.distsql.ral.queryable; +import com.sphereex.dbplusengine.SphereEx; +import com.sphereex.dbplusengine.SphereEx.Type; import lombok.SneakyThrows; import org.apache.shardingsphere.distsql.statement.ral.queryable.export.ExportDatabaseConfigurationStatement; import org.apache.shardingsphere.infra.algorithm.core.config.AlgorithmConfiguration; @@ -71,7 +73,8 @@ class ExportDatabaseConfigurationExecutorTest { when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.singleton(createShardingRuleConfiguration())); ExportDatabaseConfigurationExecutor executor = new ExportDatabaseConfigurationExecutor(); executor.setDatabase(database); - Collection<LocalDataQueryResultRow> actual = executor.getRows(new ExportDatabaseConfigurationStatement(mock(DatabaseSegment.class), null), mock(ContextManager.class)); + @SphereEx(Type.MODIFY) + Collection<LocalDataQueryResultRow> actual = executor.getRows(new ExportDatabaseConfigurationStatement(mock(DatabaseSegment.class), null, null), mock(ContextManager.class)); assertThat(actual.size(), is(1)); LocalDataQueryResultRow row = actual.iterator().next(); assertThat(row.getCell(1), is(loadExpectedRow())); @@ -94,7 +97,8 @@ class ExportDatabaseConfigurationExecutorTest { when(database.getName()).thenReturn("empty_db"); when(database.getResourceMetaData().getStorageUnits()).thenReturn(Collections.emptyMap()); when(database.getRuleMetaData().getConfigurations()).thenReturn(Collections.emptyList()); - ExportDatabaseConfigurationStatement sqlStatement = new ExportDatabaseConfigurationStatement(new DatabaseSegment(0, 0, new IdentifierValue("empty_db")), null); + @SphereEx(Type.MODIFY) + ExportDatabaseConfigurationStatement sqlStatement = new ExportDatabaseConfigurationStatement(new DatabaseSegment(0, 0, new IdentifierValue("empty_db")), null, null); ExportDatabaseConfigurationExecutor executor = new ExportDatabaseConfigurationExecutor(); executor.setDatabase(database); Collection<LocalDataQueryResultRow> actual = executor.getRows(sqlStatement, mock(ContextManager.class)); diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java index 8d0f398ccf0..bbc3f0432d1 100644 --- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java +++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportDatabaseConfigurationExecutorTest.java @@ -124,7 +124,9 @@ class ImportDatabaseConfigurationExecutorTest { init(databaseName); URL url = ImportDatabaseConfigurationExecutorTest.class.getResource(filePath); assertNotNull(url); - executor.executeUpdate(new ImportDatabaseConfigurationStatement(url.getPath()), mock(ContextManager.class)); + // SPEX CHANGED: BEGIN + executor.executeUpdate(new ImportDatabaseConfigurationStatement(null, url.getPath()), mock(ContextManager.class)); + // SPEX CHANGED: END } @SneakyThrows({IllegalAccessException.class, NoSuchFieldException.class})
