This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 7f2b189c853 Adjust the structure of pipeline IT to support starting
multiple storage containers (#23611)
7f2b189c853 is described below
commit 7f2b189c8535c275183dd899f6aa67ef7da219d6
Author: Xinze Guo <[email protected]>
AuthorDate: Tue Jan 17 17:24:45 2023 +0800
Adjust the structure of pipeline IT to support starting multiple storage
containers (#23611)
* Adjust the structure of pipeline IT to support starting multiple storage
containers
* Use default container environment
---
.../pipeline/cases/base/PipelineBaseE2EIT.java | 16 ++++++----
.../createtable/CreateTableSQLGeneratorIT.java | 12 ++++++--
.../container/compose/DockerContainerComposer.java | 36 ++++++++++++++++------
.../framework/param/PipelineTestParameter.java | 9 ++++++
.../framework/watcher/PipelineWatcher.java | 13 +++++---
.../test/resources/env/mysql/{ => mysql5}/my.cnf | 0
6 files changed, 63 insertions(+), 23 deletions(-)
diff --git
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
index 1bceb9b61f4..232f3c9d033 100644
---
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
+++
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
@@ -111,11 +111,11 @@ public abstract class PipelineBaseE2EIT {
public PipelineBaseE2EIT(final PipelineTestParameter testParam) {
databaseType = testParam.getDatabaseType();
containerComposer = ENV.getItEnvType() == PipelineEnvTypeEnum.DOCKER
- ? new DockerContainerComposer(testParam.getDatabaseType(),
testParam.getStorageContainerImage())
+ ? new DockerContainerComposer(testParam.getDatabaseType(),
testParam.getStorageContainerImage(), testParam.getStorageContainerCount())
: new NativeContainerComposer(testParam.getDatabaseType());
containerComposer.start();
if (ENV.getItEnvType() == PipelineEnvTypeEnum.DOCKER) {
- DockerStorageContainer storageContainer =
((DockerContainerComposer) containerComposer).getStorageContainer();
+ DockerStorageContainer storageContainer =
((DockerContainerComposer) containerComposer).getStorageContainers().get(0);
username = storageContainer.getUsername();
password = storageContainer.getPassword();
} else {
@@ -168,16 +168,20 @@ public abstract class PipelineBaseE2EIT {
: jdbcUrl;
}
- protected String getActualJdbcUrlTemplate(final String databaseName, final
boolean isInContainer) {
+ protected String getActualJdbcUrlTemplate(final String databaseName, final
boolean isInContainer, final int storageContainerIndex) {
if (PipelineEnvTypeEnum.DOCKER == ENV.getItEnvType()) {
- DockerStorageContainer storageContainer =
((DockerContainerComposer) containerComposer).getStorageContainer();
+ DockerStorageContainer storageContainer =
((DockerContainerComposer)
containerComposer).getStorageContainers().get(storageContainerIndex);
return isInContainer
- ? DataSourceEnvironment.getURL(getDatabaseType(),
getDatabaseType().getType().toLowerCase() + ".host",
storageContainer.getExposedPort(), databaseName)
- : DataSourceEnvironment.getURL(getDatabaseType(),
storageContainer.getHost(), storageContainer.getFirstMappedPort(),
databaseName);
+ ? DataSourceEnvironment.getURL(getDatabaseType(),
storageContainer.getNetworkAliases().get(0), storageContainer.getExposedPort(),
databaseName)
+ : storageContainer.getJdbcUrl(databaseName);
}
return DataSourceEnvironment.getURL(getDatabaseType(), "127.0.0.1",
ENV.getActualDataSourceDefaultPort(databaseType), databaseName);
}
+ protected String getActualJdbcUrlTemplate(final String databaseName, final
boolean isInContainer) {
+ return getActualJdbcUrlTemplate(databaseName, isInContainer, 0);
+ }
+
protected abstract String getSourceTableOrderName();
protected String getTargetTableOrderName() {
diff --git
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/createtable/CreateTableSQLGeneratorIT.java
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/createtable/CreateTableSQLGeneratorIT.java
index 1cdb5a0e2b2..521e924e116 100644
---
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/createtable/CreateTableSQLGeneratorIT.java
+++
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/createtable/CreateTableSQLGeneratorIT.java
@@ -53,6 +53,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
@@ -90,9 +91,14 @@ public final class CreateTableSQLGeneratorIT {
rootEntity = JAXB.unmarshal(
Objects.requireNonNull(CreateTableSQLGeneratorIT.class.getClassLoader().getResource(testParam.getScenario())),
CreateTableSQLGeneratorAssertionsRootEntity.class);
DatabaseType databaseType = testParam.getDatabaseType();
- StorageContainerConfiguration storageContainerConfig =
DatabaseTypeUtil.isMySQL(databaseType) && new
DockerImageVersion(testParam.getStorageContainerImage()).getMajorVersion() > 5
- ? MySQLContainerConfigurationFactory.newInstance(null, null,
Collections.singletonMap("/env/mysql/mysql8/my.cnf",
StorageContainerConstants.MYSQL_CONF_IN_CONTAINER))
- :
StorageContainerConfigurationFactory.newInstance(databaseType);
+ StorageContainerConfiguration storageContainerConfig;
+ if (DatabaseTypeUtil.isMySQL(databaseType)) {
+ int majorVersion = new
DockerImageVersion(testParam.getStorageContainerImage()).getMajorVersion();
+ Map<String, String> mountedResources =
Collections.singletonMap(String.format("/env/mysql/mysql%s/my.cnf",
majorVersion), StorageContainerConstants.MYSQL_CONF_IN_CONTAINER);
+ storageContainerConfig =
MySQLContainerConfigurationFactory.newInstance(null, null, mountedResources);
+ } else {
+ storageContainerConfig =
StorageContainerConfigurationFactory.newInstance(databaseType);
+ }
storageContainer = (DockerStorageContainer)
StorageContainerFactory.newInstance(databaseType,
testParam.getStorageContainerImage(), "",
storageContainerConfig);
storageContainer.start();
diff --git
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/DockerContainerComposer.java
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/DockerContainerComposer.java
index fbf2750186c..a70bb92389e 100644
---
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/DockerContainerComposer.java
+++
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/DockerContainerComposer.java
@@ -37,7 +37,11 @@ import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.im
import
org.apache.shardingsphere.test.e2e.env.container.atomic.util.DatabaseTypeUtil;
import org.apache.shardingsphere.test.e2e.env.runtime.DataSourceEnvironment;
+import java.security.InvalidParameterException;
import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
/**
* Composed container, include governance container and database container.
@@ -49,23 +53,37 @@ public final class DockerContainerComposer extends
BaseContainerComposer {
private final ShardingSphereProxyClusterContainer proxyContainer;
@Getter
- private final DockerStorageContainer storageContainer;
+ private final List<DockerStorageContainer> storageContainers = new
LinkedList<>();
@Getter
private final GovernanceContainer governanceContainer;
- public DockerContainerComposer(final DatabaseType databaseType, final
String storageContainerImage) {
+ public DockerContainerComposer(final DatabaseType databaseType, final
String storageContainerImage, final int storageContainerCount) {
this.databaseType = databaseType;
governanceContainer = getContainers().registerContainer(new
ZookeeperContainer());
- StorageContainerConfiguration storageContainerConfig =
DatabaseTypeUtil.isMySQL(databaseType) && new
DockerImageVersion(storageContainerImage).getMajorVersion() > 5
- ? MySQLContainerConfigurationFactory.newInstance(null, null,
Collections.singletonMap("/env/mysql/mysql8/my.cnf",
StorageContainerConstants.MYSQL_CONF_IN_CONTAINER))
- :
StorageContainerConfigurationFactory.newInstance(databaseType);
- storageContainer =
getContainers().registerContainer((DockerStorageContainer)
StorageContainerFactory.newInstance(databaseType, storageContainerImage,
- "", storageContainerConfig));
+ if (storageContainerCount < 1) {
+ throw new InvalidParameterException("storageContainerCount must >=
1");
+ }
+ for (int i = 0; i < storageContainerCount; i++) {
+ StorageContainerConfiguration storageContainerConfig;
+ if (DatabaseTypeUtil.isMySQL(databaseType)) {
+ int majorVersion = new
DockerImageVersion(storageContainerImage).getMajorVersion();
+ Map<String, String> mountedResources =
Collections.singletonMap(String.format("/env/mysql/mysql%s/my.cnf",
majorVersion), StorageContainerConstants.MYSQL_CONF_IN_CONTAINER);
+ storageContainerConfig =
MySQLContainerConfigurationFactory.newInstance(null, null, mountedResources);
+ } else {
+ storageContainerConfig =
StorageContainerConfigurationFactory.newInstance(databaseType);
+ }
+ DockerStorageContainer storageContainer =
getContainers().registerContainer((DockerStorageContainer)
StorageContainerFactory.newInstance(databaseType, storageContainerImage, null,
+ storageContainerConfig));
+
storageContainer.setNetworkAliases(Collections.singletonList(String.join(".",
databaseType.getType().toLowerCase() + "_" + i, "host")));
+ storageContainers.add(storageContainer);
+ }
AdaptorContainerConfiguration containerConfig =
PipelineProxyClusterContainerConfigurationFactory.newInstance(databaseType,
storageContainerImage);
ShardingSphereProxyClusterContainer proxyClusterContainer =
(ShardingSphereProxyClusterContainer)
AdapterContainerFactory.newInstance(EnvironmentConstants.CLUSTER_MODE,
- AdapterContainerConstants.PROXY, databaseType,
storageContainer, "", containerConfig);
- proxyClusterContainer.dependsOn(governanceContainer, storageContainer);
+ AdapterContainerConstants.PROXY, databaseType, null, "",
containerConfig);
+ for (DockerStorageContainer each : storageContainers) {
+ proxyClusterContainer.dependsOn(governanceContainer, each);
+ }
proxyContainer =
getContainers().registerContainer(proxyClusterContainer);
}
diff --git
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/param/PipelineTestParameter.java
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/param/PipelineTestParameter.java
index 9e20e497f54..e6ec743e71f 100644
---
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/param/PipelineTestParameter.java
+++
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/param/PipelineTestParameter.java
@@ -32,4 +32,13 @@ public final class PipelineTestParameter {
private final String storageContainerImage;
private final String scenario;
+
+ private final int storageContainerCount;
+
+ public PipelineTestParameter(final DatabaseType databaseType, final String
storageContainerImage, final String scenario) {
+ this.databaseType = databaseType;
+ this.storageContainerImage = storageContainerImage;
+ this.scenario = scenario;
+ storageContainerCount = 1;
+ }
}
diff --git
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/watcher/PipelineWatcher.java
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/watcher/PipelineWatcher.java
index 01d64c5892f..fa765e17ccb 100644
---
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/watcher/PipelineWatcher.java
+++
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/watcher/PipelineWatcher.java
@@ -20,12 +20,12 @@ package
org.apache.shardingsphere.test.e2e.data.pipeline.framework.watcher;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
-import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.BaseContainerComposer;
-import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.DockerContainerComposer;
-import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.NativeContainerComposer;
import
org.apache.shardingsphere.mode.repository.cluster.ClusterPersistRepository;
import
org.apache.shardingsphere.mode.repository.cluster.ClusterPersistRepositoryConfiguration;
import
org.apache.shardingsphere.mode.repository.cluster.zookeeper.ZookeeperRepository;
+import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.BaseContainerComposer;
+import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.DockerContainerComposer;
+import
org.apache.shardingsphere.test.e2e.data.pipeline.framework.container.compose.NativeContainerComposer;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
@@ -47,10 +47,13 @@ public class PipelineWatcher extends TestWatcher {
}
}
- // TODO now the meta data mistake is not reproduce, but keep this method,
it may be used again later
+ // TODO now the meta data mistake is not reproduce, but keep this method,
it may be used again
private void outputZookeeperData() {
+ if (!(containerComposer instanceof DockerContainerComposer)) {
+ return;
+ }
DockerContainerComposer dockerContainerComposer =
(DockerContainerComposer) containerComposer;
- DatabaseType databaseType =
dockerContainerComposer.getStorageContainer().getDatabaseType();
+ DatabaseType databaseType =
dockerContainerComposer.getStorageContainers().get(0).getDatabaseType();
String namespace = "it_db_" + databaseType.getType().toLowerCase();
ClusterPersistRepositoryConfiguration config = new
ClusterPersistRepositoryConfiguration("ZooKeeper", namespace,
dockerContainerComposer.getGovernanceContainer().getServerLists(), new
Properties());
diff --git a/test/e2e/pipeline/src/test/resources/env/mysql/my.cnf
b/test/e2e/pipeline/src/test/resources/env/mysql/mysql5/my.cnf
similarity index 100%
rename from test/e2e/pipeline/src/test/resources/env/mysql/my.cnf
rename to test/e2e/pipeline/src/test/resources/env/mysql/mysql5/my.cnf
