This is an automated email from the ASF dual-hosted git repository. zhonghongsheng pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push: new 961f25b5869 Init scaling integration test (#16954) 961f25b5869 is described below commit 961f25b5869b5d1fe3221d4e4969807e8ee16c23 Author: azexcy <101622833+aze...@users.noreply.github.com> AuthorDate: Thu Apr 21 10:28:38 2022 +0800 Init scaling integration test (#16954) * Init Scaling IT test environment * Extraction method to generate network alias * Use zk cluster * Change the base package name --- .../framework/container/atomic/ITContainers.java | 6 +- .../test/integration/util/NetworkAliasUtil.java} | 42 +++----- .../pom.xml | 11 ++ .../test/mysql => data/pipline}/ScalingIT.java | 12 +-- .../integration/data/pipline/cases/BaseITCase.java | 78 ++++++++++++++ .../pipline/cases/mysql/MySQLAutoRuleScaling.java | 64 ++++++++++++ .../data/pipline/container/ComposedContainer.java | 91 ++++++++++++++++ .../pipline/container/DockerDatabaseContainer.java | 62 +++++++++++ .../container/ShardingSphereProxyContainer.java | 114 +++++++++++++++++++++ .../pipline/container/database/MySQLContainer.java | 42 ++++++++ .../container/database/PostgreSQLContainer.java} | 33 +++--- .../pipline}/env/ITEnvironmentContext.java | 10 +- .../pipline}/env/IntegrationTestEnvironment.java | 12 +-- .../mysql => data/pipline}/env/cases/DataSet.java | 2 +- .../mysql => data/pipline}/env/cases/Type.java | 2 +- .../pipline}/env/config/SourceConfiguration.java | 4 +- .../pipline}/env/config/TargetConfiguration.java | 4 +- .../pipline/factory/DatabaseContainerFactory.java | 49 +++++++++ .../pipline}/fixture/DataImporter.java | 8 +- .../mysql => data/pipline}/util/ExecuteUtil.java | 6 +- .../test/mysql => data/pipline}/util/Executor.java | 2 +- .../mysql => data/pipline}/util/ScalingUtil.java | 2 +- .../common/proxy/conf/logback.xml} | 31 +++--- .../proxy/conf/server.yaml} | 34 ++++-- .../{engine-env.properties => it-env.properties} | 22 ++-- .../resources/env/mysql/{init.sql => initdb.sql} | 9 +- .../test/resources/env/mysql/rule_inti.properties | 34 ++++++ .../env/{mysql/init.sql => postgresql/initdb.sql} | 6 -- .../{cases/mysql/types.xml => logback-test.xml} | 31 +++--- .../compose/mode/ClusterComposedContainer.java | 14 +-- .../compose/mode/MemoryComposedContainer.java | 12 ++- 31 files changed, 708 insertions(+), 141 deletions(-) diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/framework/container/atomic/ITContainers.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/framework/container/atomic/ITContainers.java index 11eb5c2aa28..a770b9c7e96 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/framework/container/atomic/ITContainers.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/framework/container/atomic/ITContainers.java @@ -48,18 +48,16 @@ public final class ITContainers implements Startable { * Register container. * * @param container container to be registered - * @param containerType container type - * @param attachScenario whether attach scenario to container network alias + * @param networkAlias network alias * @param <T> type of ShardingSphere container * @return registered container */ - public <T extends ITContainer> T registerContainer(final T container, final String containerType, final boolean attachScenario) { + public <T extends ITContainer> T registerContainer(final T container, final String networkAlias) { if (container instanceof EmbeddedITContainer) { embeddedContainers.add((EmbeddedITContainer) container); } else { DockerITContainer dockerContainer = (DockerITContainer) container; dockerContainer.setNetwork(network); - String networkAlias = attachScenario ? String.join(".", containerType.toLowerCase(), scenario, "host") : String.join(".", containerType.toLowerCase(), "host"); dockerContainer.setNetworkAliases(Collections.singletonList(networkAlias)); String loggerName = String.join(":", scenario, dockerContainer.getName()); dockerContainer.withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(loggerName), true)); diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/util/NetworkAliasUtil.java similarity index 54% copy from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java copy to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/util/NetworkAliasUtil.java index a223f66941e..49d8d0727d7 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-env/src/test/java/org/apache/shardingsphere/test/integration/util/NetworkAliasUtil.java @@ -15,36 +15,28 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.util; +package org.apache.shardingsphere.test.integration.util; -import lombok.RequiredArgsConstructor; - -/** - * Execute util. - */ -@RequiredArgsConstructor -public final class ExecuteUtil { - - private final Executor executor; - - private final int retryCount; +public final class NetworkAliasUtil { - private final long waitMs; + /** + * Get network alias with scenario. + * + * @param containerType container type, such as "zk", "db" + * @param scenario scenario + * @return network alias + */ + public static String getNetworkAliasWithScenario(final String containerType, final String scenario) { + return String.join(".", containerType.toLowerCase(), scenario, "host"); + } /** - * Execute. + * Get network alias. * - * @return execute result + * @param containerType container type + * @return network alias */ - public boolean execute() { - int count = 0; - while (!executor.execute() && retryCount > count) { - try { - Thread.sleep(waitMs); - } catch (final InterruptedException ignored) { - } - count++; - } - return retryCount > count; + public static String getNetworkAlias(final String containerType) { + return String.join(".", containerType.toLowerCase(), "host"); } } diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml index a0772ed9ee7..078a0e2b5ca 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml @@ -41,6 +41,12 @@ <artifactId>shardingsphere-jdbc-core</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.shardingsphere</groupId> + <artifactId>shardingsphere-integration-test-env</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -51,6 +57,11 @@ <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + </dependency> </dependencies> <build> diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/ScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/ScalingIT.java similarity index 83% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/ScalingIT.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/ScalingIT.java index ae378f40e8f..eca5f7927ac 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/ScalingIT.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/ScalingIT.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql; +package org.apache.shardingsphere.integration.data.pipline; import lombok.extern.slf4j.Slf4j; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.ITEnvironmentContext; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.IntegrationTestEnvironment; -import org.apache.shardingsphere.integration.scaling.test.mysql.fixture.DataImporter; -import org.apache.shardingsphere.integration.scaling.test.mysql.util.ExecuteUtil; -import org.apache.shardingsphere.integration.scaling.test.mysql.util.ScalingUtil; +import org.apache.shardingsphere.integration.data.pipline.env.ITEnvironmentContext; +import org.apache.shardingsphere.integration.data.pipline.env.IntegrationTestEnvironment; +import org.apache.shardingsphere.integration.data.pipline.fixture.DataImporter; +import org.apache.shardingsphere.integration.data.pipline.util.ExecuteUtil; +import org.apache.shardingsphere.integration.data.pipline.util.ScalingUtil; import org.junit.Ignore; import org.junit.Test; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/BaseITCase.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/BaseITCase.java new file mode 100644 index 00000000000..3cabf98b7d4 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/BaseITCase.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.cases; + +import lombok.AccessLevel; +import lombok.Getter; +import org.apache.shardingsphere.infra.database.type.DatabaseType; +import org.apache.shardingsphere.integration.data.pipline.container.ComposedContainer; +import org.apache.shardingsphere.mode.repository.cluster.ClusterPersistRepository; +import org.apache.shardingsphere.mode.repository.cluster.ClusterPersistRepositoryConfiguration; +import org.apache.shardingsphere.mode.repository.cluster.zookeeper.CuratorZookeeperRepository; +import org.apache.shardingsphere.test.integration.framework.container.atomic.governance.GovernanceContainer; +import org.junit.Before; + +import javax.sql.DataSource; +import java.util.List; +import java.util.Properties; + +@Getter(AccessLevel.PROTECTED) +public abstract class BaseITCase { + + private DataSource targetDataSource; + + @Getter(AccessLevel.NONE) + private final ComposedContainer composedContainer; + + private String databaseNetworkAlias; + + private ClusterPersistRepository clusterPersistRepository; + + public BaseITCase(final DatabaseType databaseType) { + composedContainer = new ComposedContainer(databaseType); + } + + @Before + public void setUp() { + composedContainer.start(); + targetDataSource = composedContainer.getTargetDataSource(); + GovernanceContainer governanceContainer = composedContainer.getGovernanceContainer(); + databaseNetworkAlias = composedContainer.getDatabaseNetworkAlias(); + clusterPersistRepository = new CuratorZookeeperRepository(); + clusterPersistRepository.init(new ClusterPersistRepositoryConfiguration("ZooKeeper", "it_db", governanceContainer.getServerLists(), new Properties())); + System.out.println(clusterPersistRepository.get("/")); + } + + /** + * Query actual source database name. + * + * @return actual source database name list + */ + public List<String> listSourceDatabaseName() { + return composedContainer.listSourceDatabaseName(); + } + + /** + * Query actual target database name. + * + * @return actual target database name list + */ + public List<String> listTargetDatabaseName() { + return composedContainer.listTargetDatabaseName(); + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/mysql/MySQLAutoRuleScaling.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/mysql/MySQLAutoRuleScaling.java new file mode 100644 index 00000000000..a5cf19aeba0 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/cases/mysql/MySQLAutoRuleScaling.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.cases.mysql; + +import lombok.SneakyThrows; +import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType; +import org.apache.shardingsphere.integration.data.pipline.cases.BaseITCase; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.util.Properties; + +/** + * MySQL auto rule scaling test case. + */ +public final class MySQLAutoRuleScaling extends BaseITCase { + + public MySQLAutoRuleScaling() { + super(new MySQLDatabaseType()); + } + + @SneakyThrows + @Before + public void setUp() { + super.setUp(); + Properties initProps = new Properties(); + initProps.load(getClass().getClassLoader().getResourceAsStream("env/mysql/rule_inti.properties")); + try (Connection connection = getTargetDataSource().getConnection()) { + connection.createStatement().execute("CREATE DATABASE sharding_db;"); + connection.createStatement().execute("USE sharding_db;"); + int dbIndex = 0; + for (String value : listSourceDatabaseName()) { + connection.createStatement().execute(String.format(initProps.getProperty("add.resource.sql"), dbIndex, getDatabaseNetworkAlias(), value)); + dbIndex++; + } + for (String value : listTargetDatabaseName()) { + connection.createStatement().execute(String.format(initProps.getProperty("add.resource.sql"), dbIndex, getDatabaseNetworkAlias(), value)); + dbIndex++; + } + connection.createStatement().execute(initProps.getProperty("create.table.rule")); + connection.createStatement().execute(initProps.getProperty("create.table.sql")); + } + } + + @Test + public void test() { + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ComposedContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ComposedContainer.java new file mode 100644 index 00000000000..f591954125e --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ComposedContainer.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.container; + +import lombok.Getter; +import org.apache.shardingsphere.infra.database.type.DatabaseType; +import org.apache.shardingsphere.integration.data.pipline.factory.DatabaseContainerFactory; +import org.apache.shardingsphere.test.integration.framework.container.atomic.ITContainers; +import org.apache.shardingsphere.test.integration.framework.container.atomic.governance.GovernanceContainer; +import org.apache.shardingsphere.test.integration.framework.container.atomic.governance.GovernanceContainerFactory; +import org.apache.shardingsphere.test.integration.util.NetworkAliasUtil; +import org.testcontainers.lifecycle.Startable; + +import javax.sql.DataSource; +import java.util.List; + +public final class ComposedContainer implements Startable { + + private final ITContainers containers; + + @Getter + private final GovernanceContainer governanceContainer; + + private final DockerDatabaseContainer databaseContainer; + + private final ShardingSphereProxyContainer proxyContainer; + + @Getter + private final String databaseNetworkAlias = NetworkAliasUtil.getNetworkAlias("db"); + + public ComposedContainer(final DatabaseType databaseType) { + this.containers = new ITContainers(""); + this.governanceContainer = containers.registerContainer(GovernanceContainerFactory.newInstance("ZooKeeper"), NetworkAliasUtil.getNetworkAlias("zk")); + this.databaseContainer = containers.registerContainer(DatabaseContainerFactory.newInstance(databaseType), databaseNetworkAlias); + ShardingSphereProxyContainer proxyContainer = new ShardingSphereProxyContainer(databaseType, ""); + proxyContainer.dependsOn(governanceContainer, databaseContainer); + this.proxyContainer = containers.registerContainer(proxyContainer, NetworkAliasUtil.getNetworkAlias("sharding-proxy")); + } + + @Override + public void start() { + containers.start(); + } + + @Override + public void stop() { + containers.stop(); + } + + /** + * Get target data source. + * + * @return target data source. + */ + public DataSource getTargetDataSource() { + return proxyContainer.getTargetDataSource(); + } + + /** + * Get actual data source map. + * + * @return actual data source map + */ + public List<String> listSourceDatabaseName() { + return databaseContainer.getSourceDatabaseNames(); + } + + /** + * Get expected data source map. + * + * @return expected data source map + */ + public List<String> listTargetDatabaseName() { + return databaseContainer.getTargetDatabaseNames(); + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/DockerDatabaseContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/DockerDatabaseContainer.java new file mode 100644 index 00000000000..ae583a1a622 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/DockerDatabaseContainer.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.container; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.shardingsphere.infra.database.type.DatabaseType; +import org.apache.shardingsphere.test.integration.framework.container.atomic.DockerITContainer; +import org.testcontainers.containers.BindMode; + +import java.util.LinkedList; +import java.util.List; + +/** + * Docker storage container. + */ +@Getter +public abstract class DockerDatabaseContainer extends DockerITContainer { + + private final DatabaseType databaseType; + + private final List<String> sourceDatabaseNames; + + private final List<String> targetDatabaseNames; + + public DockerDatabaseContainer(final DatabaseType databaseType, final String dockerImageName) { + super(databaseType.getName().toLowerCase(), dockerImageName); + this.databaseType = databaseType; + sourceDatabaseNames = new LinkedList<>(); + targetDatabaseNames = new LinkedList<>(); + } + + @Override + protected void configure() { + withClasspathResourceMapping(String.format("/env/%s", databaseType.getName()), "/docker-entrypoint-initdb.d/", BindMode.READ_ONLY); + } + + @Override + @SneakyThrows + protected void postStart() { + sourceDatabaseNames.addAll(Lists.newArrayList("ds_0", "ds_1")); + targetDatabaseNames.addAll(Lists.newArrayList("ds_2", "ds_3", "ds_4")); + } + + protected abstract int getPort(); +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ShardingSphereProxyContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ShardingSphereProxyContainer.java new file mode 100644 index 00000000000..2e381a4a4f3 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/ShardingSphereProxyContainer.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.container; + +import com.zaxxer.hikari.HikariDataSource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.shardingsphere.infra.database.type.DatabaseType; +import org.apache.shardingsphere.test.integration.env.DataSourceEnvironment; +import org.apache.shardingsphere.test.integration.framework.container.atomic.DockerITContainer; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * ShardingSphere proxy container. + */ +@Slf4j +public final class ShardingSphereProxyContainer extends DockerITContainer { + + private final DatabaseType databaseType; + + private final String schemaName; + + private final AtomicReference<DataSource> targetDataSourceProvider = new AtomicReference<>(); + + public ShardingSphereProxyContainer(final DatabaseType databaseType, final String schemaName) { + super("Scaling-Proxy", "apache/shardingsphere-proxy-test"); + this.databaseType = databaseType; + this.schemaName = schemaName; + } + + @Override + protected void configure() { + mapConfigurationFiles(); + withExposedPorts(3307); + setWaitStrategy(new JDBCConnectionWaitStrategy(() -> DriverManager.getConnection(DataSourceEnvironment.getURL(databaseType, getHost(), getMappedPort(3307), ""), "root", "root"))); + } + + private void mapConfigurationFiles() { + String containerPath = "/opt/shardingsphere-proxy/conf"; + withClasspathResourceMapping("/env/common/proxy/conf/", containerPath, BindMode.READ_ONLY); + } + + /** + * Get target data source. + * + * @return target data source. + */ + public DataSource getTargetDataSource() { + DataSource dataSource = targetDataSourceProvider.get(); + if (Objects.isNull(dataSource)) { + targetDataSourceProvider.set(createProxyDataSource()); + } + return targetDataSourceProvider.get(); + } + + private DataSource createProxyDataSource() { + HikariDataSource result = new HikariDataSource(); + result.setDriverClassName(DataSourceEnvironment.getDriverClassName(databaseType)); + result.setJdbcUrl(DataSourceEnvironment.getURL(databaseType, getHost(), getMappedPort(3307), schemaName)); + result.setUsername("root"); + result.setPassword("root"); + result.setMaximumPoolSize(2); + result.setTransactionIsolation("TRANSACTION_READ_COMMITTED"); + return result; + } + + @Slf4j + @RequiredArgsConstructor + private static class JDBCConnectionWaitStrategy extends AbstractWaitStrategy { + + private final Callable<Connection> connectionSupplier; + + @Override + protected void waitUntilReady() { + Unreliables.retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { + getRateLimiter().doWhenReady(() -> { + try (Connection unused = connectionSupplier.call()) { + log.info("Container ready"); + // CHECKSTYLE:OFF + } catch (final Exception ex) { + // CHECKSTYLE:ON + throw new RuntimeException("Not Ready yet.", ex); + } + }); + return true; + }); + } + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/MySQLContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/MySQLContainer.java new file mode 100644 index 00000000000..9dfc96f4dcb --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/MySQLContainer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.container.database; + +import com.google.common.collect.Lists; +import org.apache.shardingsphere.infra.database.type.DatabaseTypeRegistry; +import org.apache.shardingsphere.integration.data.pipline.container.DockerDatabaseContainer; + +public class MySQLContainer extends DockerDatabaseContainer { + + public MySQLContainer(final String dockerImageName) { + super(DatabaseTypeRegistry.getActualDatabaseType("MySQL"), dockerImageName); + } + + @Override + protected void configure() { + super.configure(); + withExposedPorts(3306); + setEnv(Lists.newArrayList("LANG=C.UTF-8", "MYSQL_ROOT_PASSWORD=root", "MYSQL_ROOT_HOST=%", "MYSQL_DATABASE=test")); + withCommand("--sql_mode=", "--default-authentication-plugin=mysql_native_password", "--character-set-server=utf8mb4", "--collation-server=utf8mb4_unicode_ci"); + } + + @Override + protected int getPort() { + return 3306; + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/PostgreSQLContainer.java similarity index 56% copy from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java copy to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/PostgreSQLContainer.java index 04f90d09841..b226d05b15f 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/container/database/PostgreSQLContainer.java @@ -15,22 +15,25 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env.cases; +package org.apache.shardingsphere.integration.data.pipline.container.database; -import lombok.Getter; +import org.apache.shardingsphere.infra.database.type.DatabaseTypeRegistry; +import org.apache.shardingsphere.integration.data.pipline.container.DockerDatabaseContainer; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.util.LinkedList; -import java.util.List; - -/** - * Data set. - */ -@Getter -@XmlRootElement(name = "dataset") -public final class DataSet { +// TODO not complete yet +public class PostgreSQLContainer extends DockerDatabaseContainer { + + public PostgreSQLContainer(final String dockerImageName) { + super(DatabaseTypeRegistry.getActualDatabaseType("PostgreSQL"), dockerImageName); + } + + @Override + public void start() { + + } - @XmlElement(name = "type") - private final List<Type> types = new LinkedList<>(); + @Override + protected int getPort() { + return 5432; + } } diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/ITEnvironmentContext.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/ITEnvironmentContext.java similarity index 91% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/ITEnvironmentContext.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/ITEnvironmentContext.java index bf2fbe1a44c..fb8db92a097 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/ITEnvironmentContext.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/ITEnvironmentContext.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env; +package org.apache.shardingsphere.integration.data.pipline.env; import com.google.gson.Gson; import lombok.Getter; @@ -25,10 +25,10 @@ import org.apache.shardingsphere.data.pipeline.api.config.rulealtered.PipelineCo import org.apache.shardingsphere.data.pipeline.api.datasource.config.PipelineDataSourceConfiguration; import org.apache.shardingsphere.data.pipeline.api.datasource.config.yaml.YamlPipelineDataSourceConfiguration; import org.apache.shardingsphere.data.pipeline.core.datasource.PipelineDataSourceFactory; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.cases.DataSet; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.cases.Type; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.config.SourceConfiguration; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.config.TargetConfiguration; +import org.apache.shardingsphere.integration.data.pipline.env.cases.DataSet; +import org.apache.shardingsphere.integration.data.pipline.env.cases.Type; +import org.apache.shardingsphere.integration.data.pipline.env.config.SourceConfiguration; +import org.apache.shardingsphere.integration.data.pipline.env.config.TargetConfiguration; import org.apache.shardingsphere.sharding.yaml.config.rule.YamlTableRuleConfiguration; import javax.sql.DataSource; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/IntegrationTestEnvironment.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/IntegrationTestEnvironment.java similarity index 84% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/IntegrationTestEnvironment.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/IntegrationTestEnvironment.java index ad64d431df1..d0138d2deb1 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/IntegrationTestEnvironment.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/IntegrationTestEnvironment.java @@ -15,13 +15,12 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env; +package org.apache.shardingsphere.integration.data.pipline.env; import lombok.Getter; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.shardingsphere.integration.scaling.test.mysql.util.ScalingUtil; -import org.apache.shardingsphere.integration.scaling.test.mysql.util.ExecuteUtil; +import org.apache.shardingsphere.integration.data.pipline.util.ExecuteUtil; +import org.apache.shardingsphere.integration.data.pipline.util.ScalingUtil; import java.io.IOException; import java.io.InputStream; @@ -37,10 +36,9 @@ public final class IntegrationTestEnvironment { private final Properties props; - @SneakyThrows private IntegrationTestEnvironment() { - props = loadProperties("env/engine-env.properties"); - isEnvironmentPrepared = props.getProperty("it.env.value").equals(props.getProperty("it.env.type")); + props = loadProperties("env/it-env.properties"); + isEnvironmentPrepared = false; } private Properties loadProperties(final String propsFileName) { diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/DataSet.java similarity index 93% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/DataSet.java index 04f90d09841..81d85910364 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/DataSet.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/DataSet.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env.cases; +package org.apache.shardingsphere.integration.data.pipline.env.cases; import lombok.Getter; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/Type.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/Type.java similarity index 96% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/Type.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/Type.java index d7a7b7c3595..3e060f1668e 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/cases/Type.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/cases/Type.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env.cases; +package org.apache.shardingsphere.integration.data.pipline.env.cases; import lombok.Getter; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/SourceConfiguration.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/SourceConfiguration.java similarity index 96% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/SourceConfiguration.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/SourceConfiguration.java index 504dbc3c867..dc6a6d7abe6 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/SourceConfiguration.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/SourceConfiguration.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env.config; +package org.apache.shardingsphere.integration.data.pipline.env.config; import lombok.SneakyThrows; import org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource; @@ -24,7 +24,7 @@ import org.apache.shardingsphere.infra.yaml.config.pojo.YamlRootConfiguration; import org.apache.shardingsphere.infra.yaml.config.swapper.YamlDataSourceConfigurationSwapper; import org.apache.shardingsphere.infra.yaml.config.swapper.YamlRuleConfigurationSwapperEngine; import org.apache.shardingsphere.infra.yaml.engine.YamlEngine; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.IntegrationTestEnvironment; +import org.apache.shardingsphere.integration.data.pipline.env.IntegrationTestEnvironment; import org.apache.shardingsphere.data.pipeline.api.datasource.config.impl.ShardingSpherePipelineDataSourceConfiguration; import org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration; import org.apache.shardingsphere.sharding.yaml.config.rule.YamlTableRuleConfiguration; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/TargetConfiguration.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/TargetConfiguration.java similarity index 92% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/TargetConfiguration.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/TargetConfiguration.java index ad78c1f7f5c..478ca2b7c45 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/env/config/TargetConfiguration.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/env/config/TargetConfiguration.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.env.config; +package org.apache.shardingsphere.integration.data.pipline.env.config; import org.apache.shardingsphere.data.pipeline.api.datasource.config.impl.StandardPipelineDataSourceConfiguration; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.IntegrationTestEnvironment; +import org.apache.shardingsphere.integration.data.pipline.env.IntegrationTestEnvironment; import java.util.Properties; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/factory/DatabaseContainerFactory.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/factory/DatabaseContainerFactory.java new file mode 100644 index 00000000000..ab0a9c938e9 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/factory/DatabaseContainerFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.integration.data.pipline.factory; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.apache.shardingsphere.infra.database.type.DatabaseType; +import org.apache.shardingsphere.integration.data.pipline.container.DockerDatabaseContainer; +import org.apache.shardingsphere.integration.data.pipline.container.database.MySQLContainer; +import org.apache.shardingsphere.integration.data.pipline.container.database.PostgreSQLContainer; + +/** + * Storage container factory. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class DatabaseContainerFactory { + + /** + * Create new instance of storage container. + * + * @param databaseType database type + * @return new instance of storage container + */ + public static DockerDatabaseContainer newInstance(final DatabaseType databaseType) { + switch (databaseType.getName()) { + case "MySQL": + return new MySQLContainer("mysql:5.7"); + case "PostgreSQL": + return new PostgreSQLContainer("postgres:12.6"); + default: + throw new RuntimeException(String.format("Database [%s] is unknown.", databaseType.getName())); + } + } +} diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/fixture/DataImporter.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/fixture/DataImporter.java similarity index 90% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/fixture/DataImporter.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/fixture/DataImporter.java index 2d299821a39..07013bbc20f 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/fixture/DataImporter.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/fixture/DataImporter.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.fixture; +package org.apache.shardingsphere.integration.data.pipline.fixture; import lombok.SneakyThrows; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.ITEnvironmentContext; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.cases.DataSet; -import org.apache.shardingsphere.integration.scaling.test.mysql.env.cases.Type; +import org.apache.shardingsphere.integration.data.pipline.env.ITEnvironmentContext; +import org.apache.shardingsphere.integration.data.pipline.env.cases.DataSet; +import org.apache.shardingsphere.integration.data.pipline.env.cases.Type; import javax.sql.DataSource; import java.sql.Connection; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ExecuteUtil.java similarity index 89% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ExecuteUtil.java index a223f66941e..882f497b604 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ExecuteUtil.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ExecuteUtil.java @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.util; +package org.apache.shardingsphere.integration.data.pipline.util; import lombok.RequiredArgsConstructor; +import java.util.concurrent.TimeUnit; + /** * Execute util. */ @@ -40,7 +42,7 @@ public final class ExecuteUtil { int count = 0; while (!executor.execute() && retryCount > count) { try { - Thread.sleep(waitMs); + TimeUnit.MILLISECONDS.sleep(waitMs); } catch (final InterruptedException ignored) { } count++; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/Executor.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/Executor.java similarity index 93% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/Executor.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/Executor.java index 9c718c9a5af..99241539caa 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/Executor.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/Executor.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.util; +package org.apache.shardingsphere.integration.data.pipline.util; /** * Executor. diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ScalingUtil.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ScalingUtil.java similarity index 96% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ScalingUtil.java rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ScalingUtil.java index fb9dcca2961..d1015aca84a 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/scaling/test/mysql/util/ScalingUtil.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipline/util/ScalingUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shardingsphere.integration.scaling.test.mysql.util; +package org.apache.shardingsphere.integration.data.pipline.util; import com.google.gson.JsonElement; import lombok.AccessLevel; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/logback.xml similarity index 61% copy from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml copy to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/logback.xml index 220926b5ca6..1494746120c 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/logback.xml @@ -1,3 +1,4 @@ +<?xml version="1.0"?> <!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more ~ contributor license agreements. See the NOTICE file distributed with @@ -15,18 +16,18 @@ ~ limitations under the License. --> -<dataset> - <type name="int"> - <value>0</value> - <value>-2147483648</value> - <value>2147483647</value> - </type> - <type name="varchar(1)"> - <value></value> - <value>1</value> - </type> - <type name="date"> - <value>1000-01-01</value> - <value>9999-12-31</value> - </type> -</dataset> +<configuration> + <appender name="console" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{36} - %msg%n</pattern> + </encoder> + </appender> + <logger name="org.apache.shardingsphere" level="info" additivity="false"> + <appender-ref ref="console" /> + </logger> + + <root> + <level value="info" /> + <appender-ref ref="console" /> + </root> +</configuration> diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/server.yaml similarity index 56% copy from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties copy to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/server.yaml index c0516fc677e..ce314b28fa3 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/proxy/conf/server.yaml @@ -15,12 +15,30 @@ # limitations under the License. # -it.env.type=${it.env} -it.env.value=scaling +mode: + type: Cluster + repository: + type: ZooKeeper + props: + namespace: it_db + server-lists: zk.host:2181 + timeToLiveSeconds: 60 + operationTimeoutMilliseconds: 500 + retryIntervalMilliseconds: 500 + maxRetries: 3 + overwrite: false -db.host.docker=mysql:3306 -db.host.host=127.0.0.1:13306 -db.username=root -db.password=123456 -scaling.retry=30 -scaling.waitMs=1000 +rules: + - !AUTHORITY + users: + - root@:root + provider: + type: ALL_PRIVILEGES_PERMITTED + +props: + max-connections-size-per-query: 1 + kernel-executor-size: 16 # Infinite by default. + proxy-frontend-flush-threshold: 128 # The default value is 128. + proxy-hint-enabled: true + sql-show: false + sql-federation-enabled: true diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/it-env.properties similarity index 74% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/it-env.properties index c0516fc677e..aa594cdf669 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/engine-env.properties +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/it-env.properties @@ -15,12 +15,20 @@ # limitations under the License. # -it.env.type=${it.env} -it.env.value=scaling +#it.modes=Memory,Cluster +it.run.modes=Memory + +it.scenarios=scaling + +# it.cluster.env.type=DOCKER,NATIVE +it.cluster.env.type=DOCKER + +# it.cluster.adapters=jdbc,proxy +it.cluster.adapters=proxy + +# it.cluster.databases=MySQL,PostgreSQL +it.cluster.databases=MySQL + +scaling.waitMs=5000 -db.host.docker=mysql:3306 -db.host.host=127.0.0.1:13306 -db.username=root -db.password=123456 scaling.retry=30 -scaling.waitMs=1000 diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/initdb.sql similarity index 77% copy from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql copy to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/initdb.sql index 1254957fa40..e36f72acd0d 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/initdb.sql @@ -15,8 +15,11 @@ -- limitations under the License. -- -CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY '123456'; +CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY 'root'; GRANT All privileges ON *.* TO 'root'@'%'; -CREATE DATABASE ds_src; -CREATE DATABASE ds_dst; +CREATE DATABASE IF NOT EXISTS ds_0; +CREATE DATABASE IF NOT EXISTS ds_1; +CREATE DATABASE IF NOT EXISTS ds_2; +CREATE DATABASE IF NOT EXISTS ds_3; +CREATE DATABASE IF NOT EXISTS ds_4; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/rule_inti.properties b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/rule_inti.properties new file mode 100644 index 00000000000..07eabd79499 --- /dev/null +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/rule_inti.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +create.table.rule=CREATE SHARDING TABLE RULE t_order(RESOURCES(ds_0,ds_1),SHARDING_COLUMN=order_id,TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=4)),KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))), t_order_item(RESOURCES(ds_0,ds_1),SHARDING_COLUMN=order_id,TYPE(NAME=hash_mod,PROPERTIES("sharding-count"=4)),KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))); +create.table.sql=CREATE TABLE `t_order` (\ +`id` BIGINT NOT NULL COMMENT 'pk id',\ +`t_varchar` VARCHAR ( 255 ) NOT NULL,\ +`t_tinyint` TINYINT ( 1 ) NULL,\ +`t_int` INT NULL,\ +`t_timestamp` TIMESTAMP NULL,\ +`t_datetime` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP,\ +`t_binary` BINARY NULL,\ +`t_blob` BLOB NULL,\ +`t_decimal` DECIMAL ( 10, 2 ) NULL,\ +`t_char` CHAR ( 128 ) NULL,\ +`t_double` DOUBLE NULL,\ +`t_json` json NULL COMMENT 'json test',\ +PRIMARY KEY ( `id` ),\ +UNIQUE INDEX ( `t_varchar` ),\ +INDEX ( `t_int` ) ); +add.resource.sql=ADD RESOURCE ds_%s (URL="jdbc:mysql://%s:3306/%s?serverTimezone=UTC&useSSL=false",USER=root,PASSWORD=root) diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/postgresql/initdb.sql similarity index 83% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/postgresql/initdb.sql index 1254957fa40..e3b8a181bea 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/mysql/init.sql +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/postgresql/initdb.sql @@ -14,9 +14,3 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - -CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY '123456'; -GRANT All privileges ON *.* TO 'root'@'%'; - -CREATE DATABASE ds_src; -CREATE DATABASE ds_dst; diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml similarity index 58% rename from shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml rename to shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml index 220926b5ca6..d80d16d1513 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/cases/mysql/types.xml +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml @@ -1,3 +1,4 @@ +<?xml version="1.0"?> <!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more ~ contributor license agreements. See the NOTICE file distributed with @@ -15,18 +16,18 @@ ~ limitations under the License. --> -<dataset> - <type name="int"> - <value>0</value> - <value>-2147483648</value> - <value>2147483647</value> - </type> - <type name="varchar(1)"> - <value></value> - <value>1</value> - </type> - <type name="date"> - <value>1000-01-01</value> - <value>9999-12-31</value> - </type> -</dataset> +<configuration> + <appender name="console" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{36} - %msg%n</pattern> + </encoder> + </appender> + <logger name="org.apache.shardingsphere" level="warn" additivity="true"> + <appender-ref ref="console" /> + </logger> + <logger name="com.zaxxer.hikari.pool.ProxyConnection" level="off" /> + <root> + <level value="warn" /> + <appender-ref ref="console" /> + </root> +</configuration> diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/ClusterComposedContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/ClusterComposedContainer.java index 139cd6c19b9..6465cba7feb 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/ClusterComposedContainer.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/ClusterComposedContainer.java @@ -27,6 +27,7 @@ import org.apache.shardingsphere.test.integration.framework.container.atomic.sto import org.apache.shardingsphere.test.integration.framework.container.atomic.storage.StorageContainerFactory; import org.apache.shardingsphere.test.integration.framework.container.compose.ComposedContainer; import org.apache.shardingsphere.test.integration.framework.param.model.ParameterizedArray; +import org.apache.shardingsphere.test.integration.util.NetworkAliasUtil; import javax.sql.DataSource; import java.util.Map; @@ -45,13 +46,14 @@ public final class ClusterComposedContainer implements ComposedContainer { private final AdapterContainer adapterContainer; public ClusterComposedContainer(final ParameterizedArray parameterizedArray) { - containers = new ITContainers(parameterizedArray.getScenario()); + String scenario = parameterizedArray.getScenario(); + containers = new ITContainers(scenario); // TODO support other types of governance - governanceContainer = containers.registerContainer(GovernanceContainerFactory.newInstance("ZooKeeper"), "zk", false); - storageContainer = containers.registerContainer(StorageContainerFactory.newInstance( - parameterizedArray.getDatabaseType(), parameterizedArray.getScenario()), parameterizedArray.getDatabaseType().getName(), true); - adapterContainer = containers.registerContainer(AdapterContainerFactory.newInstance( - parameterizedArray.getAdapter(), parameterizedArray.getDatabaseType(), storageContainer, parameterizedArray.getScenario()), parameterizedArray.getAdapter(), true); + governanceContainer = containers.registerContainer(GovernanceContainerFactory.newInstance("ZooKeeper"), NetworkAliasUtil.getNetworkAlias("zk")); + storageContainer = containers.registerContainer(StorageContainerFactory.newInstance(parameterizedArray.getDatabaseType(), scenario), + NetworkAliasUtil.getNetworkAliasWithScenario(parameterizedArray.getDatabaseType().getName(), scenario)); + adapterContainer = containers.registerContainer(AdapterContainerFactory.newInstance(parameterizedArray.getAdapter(), parameterizedArray.getDatabaseType(), storageContainer, scenario), + NetworkAliasUtil.getNetworkAliasWithScenario(parameterizedArray.getAdapter(), scenario)); if (adapterContainer instanceof DockerITContainer) { ((DockerITContainer) adapterContainer).dependsOn(governanceContainer, storageContainer); } diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/MemoryComposedContainer.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/MemoryComposedContainer.java index a4bfcbf32f8..758a9b9cf8d 100644 --- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/MemoryComposedContainer.java +++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/src/test/java/org/apache/shardingsphere/test/integration/framework/container/compose/mode/MemoryComposedContainer.java @@ -25,6 +25,7 @@ import org.apache.shardingsphere.test.integration.framework.container.atomic.sto import org.apache.shardingsphere.test.integration.framework.container.atomic.storage.StorageContainerFactory; import org.apache.shardingsphere.test.integration.framework.container.compose.ComposedContainer; import org.apache.shardingsphere.test.integration.framework.param.model.ParameterizedArray; +import org.apache.shardingsphere.test.integration.util.NetworkAliasUtil; import javax.sql.DataSource; import java.util.Map; @@ -41,11 +42,12 @@ public final class MemoryComposedContainer implements ComposedContainer { private final AdapterContainer adapterContainer; public MemoryComposedContainer(final ParameterizedArray parameterizedArray) { - containers = new ITContainers(parameterizedArray.getScenario()); - storageContainer = containers.registerContainer(StorageContainerFactory.newInstance( - parameterizedArray.getDatabaseType(), parameterizedArray.getScenario()), parameterizedArray.getDatabaseType().getName(), true); - adapterContainer = containers.registerContainer(AdapterContainerFactory.newInstance( - parameterizedArray.getAdapter(), parameterizedArray.getDatabaseType(), storageContainer, parameterizedArray.getScenario()), parameterizedArray.getAdapter(), true); + String scenario = parameterizedArray.getScenario(); + containers = new ITContainers(scenario); + storageContainer = containers.registerContainer(StorageContainerFactory.newInstance(parameterizedArray.getDatabaseType(), scenario), + NetworkAliasUtil.getNetworkAliasWithScenario(parameterizedArray.getDatabaseType().getName(), scenario)); + adapterContainer = containers.registerContainer(AdapterContainerFactory.newInstance(parameterizedArray.getAdapter(), parameterizedArray.getDatabaseType(), storageContainer, scenario), + NetworkAliasUtil.getNetworkAliasWithScenario(parameterizedArray.getAdapter(), scenario)); if (adapterContainer instanceof DockerITContainer) { ((DockerITContainer) adapterContainer).dependsOn(storageContainer); }