This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch testcontainer in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bdf3150bd5824d36fe7cc6076928045928bda97e Author: xiangdong huang <[email protected]> AuthorDate: Wed Apr 7 01:35:58 2021 +0800 try test container --- .../resources/conf/iotdb-cluster.properties | 6 +- .../apache/iotdb/cluster/config/ClusterConfig.java | 24 ++++- docker/src/main/{Dockerfile => Dockerfile-cluster} | 4 +- docker/src/main/{Dockerfile => Dockerfile-single} | 2 +- pom.xml | 36 +++++++ .../src/test/resources/testcontainers.properties | 17 +++ test/e2e/base/docker-compose.yaml | 4 +- testcontainer/pom.xml | 118 +++++++++++++++++++++ .../java/org/apache/iotdb/db/sql/ClusterE2E.java | 80 ++++++++++++++ .../org/apache/iotdb/db/sql/SingleNodeE2E.java | 85 +++++++++++++++ .../NoProjectNameDockerComposeContainer.java | 38 +++++++ .../src/test/resources/1node}/docker-compose.yaml | 30 ++---- .../test/resources/1node/iotdb-cluster.properties | 42 +++----- .../src/test/resources/3nodes}/docker-compose.yaml | 30 ++---- .../test/resources/3nodes/iotdb-cluster.properties | 42 +++----- .../src/test/resources/5nodes}/docker-compose.yaml | 31 ++---- .../test/resources/5nodes/iotdb-cluster.properties | 42 +++----- .../src/test/resources/iotdb-engine.properties | 24 +++++ 18 files changed, 510 insertions(+), 145 deletions(-) diff --git a/cluster/src/assembly/resources/conf/iotdb-cluster.properties b/cluster/src/assembly/resources/conf/iotdb-cluster.properties index 73e7b42..3819dcc 100644 --- a/cluster/src/assembly/resources/conf/iotdb-cluster.properties +++ b/cluster/src/assembly/resources/conf/iotdb-cluster.properties @@ -29,6 +29,7 @@ #-------------------------------------------IMPORTANT---------------------------------------------# # used for communication between cluster nodes, eg heartbeatăraft logs and snapshots etc. +# if this parameter is commented, then the IP that binded by the hostname will be used. internal_ip=127.0.0.1 # port for metadata service @@ -51,7 +52,8 @@ open_server_rpc_port=false # nodes that already in the cluster, unnecessary to be the nodes that were used to build the # initial cluster by start-node.sh(.bat). Several nodes will be picked randomly to send the # request, the number of nodes picked depends on the number of retries. -seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007 +#seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007 +seed_nodes=127.0.0.1:9003 # whether to use thrift compressed protocol for internal communications. If you want to change # compression settings for external clients, please modify 'rpc_thrift_compression_enable' in @@ -65,7 +67,7 @@ rpc_thrift_compression_enable=false max_concurrent_client_num=10000 # number of replications for one partition -default_replica_num=3 +default_replica_num=1 # cluster name to identify different clusters # all node's cluster_name in one cluster are the same diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java index 11cedc8..2f4215e 100644 --- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java @@ -21,27 +21,31 @@ package org.apache.iotdb.cluster.config; import org.apache.iotdb.cluster.utils.ClusterConsistent; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; public class ClusterConfig { - + private static Logger logger = LoggerFactory.getLogger(ClusterConfig.class); static final String CONFIG_NAME = "iotdb-cluster.properties"; - private String internalIp = "127.0.0.1"; + private String internalIp; private int internalMetaPort = 9003; private int internalDataPort = 40010; private int clusterRpcPort = IoTDBDescriptor.getInstance().getConfig().getRpcPort(); /** each one is a {internalIp | domain name}:{meta port} string tuple. */ - private List<String> seedNodeUrls = - Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort)); + private List<String> seedNodeUrls; @ClusterConsistent private boolean isRpcThriftCompressionEnabled = false; private int maxConcurrentClientNum = 10000; - @ClusterConsistent private int replicationNum = 3; + @ClusterConsistent private int replicationNum = 1; @ClusterConsistent private String clusterName = "default"; @@ -164,6 +168,16 @@ public class ClusterConfig { private boolean openServerRpcPort = false; + public ClusterConfig() { + try { + internalIp = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + logger.error(e.getMessage()); + internalIp = "127.0.0.1"; + } + seedNodeUrls = Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort)); + } + public int getSelectorNumOfClientPool() { return selectorNumOfClientPool; } diff --git a/docker/src/main/Dockerfile b/docker/src/main/Dockerfile-cluster similarity index 92% copy from docker/src/main/Dockerfile copy to docker/src/main/Dockerfile-cluster index 402227c..277132b 100644 --- a/docker/src/main/Dockerfile +++ b/docker/src/main/Dockerfile-cluster @@ -21,7 +21,7 @@ FROM openjdk:11-jre-slim -ADD distribution/target/apache-iotdb-*-all-bin.zip / +ADD distribution/target/apache-iotdb-*-cluster-bin.zip / RUN apt update \ && apt install lsof procps unzip -y \ @@ -40,4 +40,4 @@ EXPOSE 8181 VOLUME /iotdb/data VOLUME /iotdb/logs ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}" -ENTRYPOINT ["/iotdb/sbin/start-server.sh"] +ENTRYPOINT ["/iotdb/sbin/start-node.sh"] diff --git a/docker/src/main/Dockerfile b/docker/src/main/Dockerfile-single similarity index 95% copy from docker/src/main/Dockerfile copy to docker/src/main/Dockerfile-single index 402227c..c0e00a9 100644 --- a/docker/src/main/Dockerfile +++ b/docker/src/main/Dockerfile-single @@ -21,7 +21,7 @@ FROM openjdk:11-jre-slim -ADD distribution/target/apache-iotdb-*-all-bin.zip / +ADD distribution/target/apache-iotdb-*-server-bin.zip / RUN apt update \ && apt install lsof procps unzip -y \ diff --git a/pom.xml b/pom.xml index 1c2b0f3..eb27a42 100644 --- a/pom.xml +++ b/pom.xml @@ -508,6 +508,12 @@ <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <version>1.15.2</version> + <scope>test</scope> + </dependency> </dependencies> <build> <pluginManagement> @@ -898,6 +904,7 @@ <excludes> <!-- Exclude integration tests within (unit) test phase. --> <exclude>src/test/**/*IT.java</exclude> + <exclude>src/test/**/*E2E.java</exclude> </excludes> </configuration> </execution> @@ -911,7 +918,12 @@ <includes> <!-- Include integration tests within integration-test phase. --> <include>src/test/**/*IT.java</include> + <include>src/test/**/*E2E.java</include> </includes> + <excludes> + <!-- Exclude unit tests within (unit) test phase. --> + <exclude>src/test/**/*Test.java</exclude> + </excludes> </configuration> </execution> </executions> @@ -991,6 +1003,30 @@ <thrift.exec-cmd.args>+x ${project.build.directory}/tools/${thrift.executable}</thrift.exec-cmd.args> </properties> </profile> + <!-- for TestContainer. As it requires docker, we have to detect whether docker exists.--> + <profile> + <!-- Mac and Unix--> + <id>unixDockerCheck</id> + <activation> + <file> + <exists>/var/run/docker.sock</exists> + </file> + </activation> + <modules> + <module>testcontainer</module> + </modules> + </profile> + <profile> + <id>WinDockerCheck</id> + <activation> + <file> + <exists>C:\Program Files\Docker\Docker\resources\bin\docker.exe</exists> + </file> + </activation> + <modules> + <module>testcontainer</module> + </modules> + </profile> <!-- Some APIs were removed in Java 11, so we need to add replacements --> <profile> <id>java-11-and-above</id> diff --git a/server/src/test/resources/testcontainers.properties b/server/src/test/resources/testcontainers.properties new file mode 100644 index 0000000..af6effe --- /dev/null +++ b/server/src/test/resources/testcontainers.properties @@ -0,0 +1,17 @@ +#Licensed to the Apache Software Foundation (ASF) under one +#or more contributor license agreements. See the NOTICE file +#distributed with this work for additional information +#regarding copyright ownership. The ASF licenses this file +#to you under the Apache License, Version 2.0 (the +#"License"); you may not use this file except in compliance +#with the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, +#software distributed under the License is distributed on an +#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +#KIND, either express or implied. See the License for the +#specific language governing permissions and limitations +#under the License. + diff --git a/test/e2e/base/docker-compose.yaml b/test/e2e/base/docker-compose.yaml index d4970d4..08ebc3b 100644 --- a/test/e2e/base/docker-compose.yaml +++ b/test/e2e/base/docker-compose.yaml @@ -23,7 +23,7 @@ services: server-prototype: build: context: ../../.. - dockerfile: docker/src/main/Dockerfile + dockerfile: ../../../docker/src/main/Dockerfile-single ports: - 6667:6667 networks: @@ -37,7 +37,7 @@ services: initializer: build: context: ../../.. - dockerfile: docker/src/main/Dockerfile + dockerfile: ../../../docker/src/main/Dockerfile-single networks: iotdb: entrypoint: diff --git a/testcontainer/pom.xml b/testcontainer/pom.xml new file mode 100644 index 0000000..f8bb9f6 --- /dev/null +++ b/testcontainer/pom.xml @@ -0,0 +1,118 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-parent</artifactId> + <version>0.12.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>testcontainer</artifactId> + <properties> + <docker.test.skip>false</docker.test.skip> + <docker.build.executable>docker</docker.build.executable> + <docker.build.single.argument>build -t apache/iotdb:maven-development -f ${basedir}/../docker/src/main/Dockerfile-single ${basedir}/../.</docker.build.single.argument> + <docker.clean.single.argument>image rm apache/iotdb:maven-development</docker.clean.single.argument> + <docker.build.cluster.argument>build -t apache/iotdb:cluster-maven-development -f ${basedir}/../docker/src/main/Dockerfile-cluster ${basedir}/../.</docker.build.cluster.argument> + <docker.clean.cluster.argument>image rm apache/iotdb:cluster-maven-development</docker.clean.cluster.argument> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-jdbc</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-cli</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <!-- before integration test, we build the docker image --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <executions> + <execution> + <id>build-docker-image</id> + <phase>pre-integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${docker.test.skip}</skip> + <executable>${docker.build.executable}</executable> + <commandlineArgs>${docker.build.single.argument}</commandlineArgs> + </configuration> + </execution> + <execution> + <id>build-cluster-docker-image</id> + <phase>pre-integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${docker.test.skip}</skip> + <executable>${docker.build.executable}</executable> + <commandlineArgs>${docker.build.cluster.argument}</commandlineArgs> + </configuration> + </execution> + <execution> + <id>clean-docker-image</id> + <phase>post-integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${docker.test.skip}</skip> + <executable>${docker.build.executable}</executable> + <commandlineArgs>${docker.clean.single.argument}</commandlineArgs> + </configuration> + </execution> + <execution> + <id>clean-cluster-docker-image</id> + <phase>post-integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${docker.test.skip}</skip> + <executable>${docker.build.executable}</executable> + <commandlineArgs>${docker.clean.cluster.argument}</commandlineArgs> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>3.2.0</version> + <executions> + <execution> + <id>add-test-container-source</id> + <phase>generate-test-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>${basedir}/src/test/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java new file mode 100644 index 0000000..44c0d08 --- /dev/null +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.sql; + +import org.apache.iotdb.jdbc.Config; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.NoProjectNameDockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class ClusterE2E { + private Statement statement; + private Connection connection; + + @Rule + public DockerComposeContainer environment = + new NoProjectNameDockerComposeContainer( + "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml")) + .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) + .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort()) + .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort()) + .withLocalCompose(true); + + int rpcPort = 6667; + + @Before + public void setUp() throws Exception { + + String ip = environment.getServiceHost("iotdb-server_1", 6667); + rpcPort = environment.getServicePort("iotdb-server_1", 6667); + + Class.forName(Config.JDBC_DRIVER_NAME); + connection = DriverManager.getConnection("jdbc:iotdb://" + ip + ":" + rpcPort, "root", "root"); + statement = connection.createStatement(); + } + + @After + public void tearDown() throws Exception { + statement.close(); + connection.close(); + } + + @Test + public void testSimplePutAndGet() throws SQLException { + String[] timeSeriesArray = {"root.sg1.aa.bb", "root.sg1.aa.bb.cc", "root.sg1.aa"}; + + for (String timeSeries : timeSeriesArray) { + statement.execute( + String.format( + "create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY", + timeSeries)); + } + } +} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java new file mode 100644 index 0000000..7a4705a --- /dev/null +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.sql; + +import org.apache.iotdb.jdbc.Config; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class SingleNodeE2E { + private Statement statement; + private Connection connection; + + @Rule + public GenericContainer dslContainer = + new GenericContainer(DockerImageName.parse("apache/iotdb:maven-development")) + .withImagePullPolicy(PullPolicy.defaultPolicy()) + // mount another properties for changing parameters, e.g., open 5555 port (sync module) + .withFileSystemBind( + new File("src/test/resources/iotdb-engine.properties").getAbsolutePath(), + "/iotdb/conf/iotdb-engine.properties", + BindMode.READ_ONLY) + .withExposedPorts(6667) + .waitingFor(Wait.forListeningPort()); + + int rpcPort = 6667; + int syncPort = 5555; + + @Before + public void setUp() throws Exception { + rpcPort = dslContainer.getMappedPort(6667); + + syncPort = dslContainer.getMappedPort(5555); + Class.forName(Config.JDBC_DRIVER_NAME); + connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:" + rpcPort, "root", "root"); + statement = connection.createStatement(); + } + + @After + public void tearDown() throws Exception { + statement.close(); + connection.close(); + } + + @Test + public void testSimplePutAndGet() throws SQLException { + String[] timeSeriesArray = {"root.sg1.aa.bb", "root.sg1.aa.bb.cc", "root.sg1.aa"}; + + for (String timeSeries : timeSeriesArray) { + statement.execute( + String.format( + "create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY", + timeSeries)); + } + } +} diff --git a/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java new file mode 100644 index 0000000..0d1fa76 --- /dev/null +++ b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.testcontainers.containers; + +import java.io.File; +import java.lang.reflect.Field; + +public class NoProjectNameDockerComposeContainer extends DockerComposeContainer { + + public NoProjectNameDockerComposeContainer(String identifier, File... composeFiles) { + super(identifier, composeFiles); + Field project = null; + try { + project = DockerComposeContainer.class.getDeclaredField("project"); + project.setAccessible(true); + project.set(this, ""); + } catch (NoSuchFieldException | IllegalAccessException e) { + e.printStackTrace(); + } + } +} diff --git a/test/e2e/base/docker-compose.yaml b/testcontainer/src/test/resources/1node/docker-compose.yaml similarity index 68% copy from test/e2e/base/docker-compose.yaml copy to testcontainer/src/test/resources/1node/docker-compose.yaml index d4970d4..9803930 100644 --- a/test/e2e/base/docker-compose.yaml +++ b/testcontainer/src/test/resources/1node/docker-compose.yaml @@ -20,32 +20,24 @@ version: '3.8' services: - server-prototype: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - ports: - - 6667:6667 + iotdb-server: + image: apache/iotdb:cluster-maven-development + expose: + - 6667 + - 9003 + - 40010 networks: - iotdb: + - iotdb healthcheck: test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] interval: 5s timeout: 60s retries: 120 + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 1 - initializer: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - networks: - iotdb: - entrypoint: - - bash - - -c - - | - cat /res/init.sql | grep -v '^--' | xargs -I {} /iotdb/sbin/start-cli.sh -h server -e {} - echo "Ready to Run IoTDB E2E Tests" networks: iotdb: + driver: bridge diff --git a/docker/src/main/Dockerfile b/testcontainer/src/test/resources/1node/iotdb-cluster.properties similarity index 57% copy from docker/src/main/Dockerfile copy to testcontainer/src/test/resources/1node/iotdb-cluster.properties index 402227c..2df52cc 100644 --- a/docker/src/main/Dockerfile +++ b/testcontainer/src/test/resources/1node/iotdb-cluster.properties @@ -15,29 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -# docker build context is the root path of the repository - -FROM openjdk:11-jre-slim - -ADD distribution/target/apache-iotdb-*-all-bin.zip / - -RUN apt update \ - && apt install lsof procps unzip -y \ - && unzip /apache-iotdb-*-bin.zip -d / \ - && rm /apache-iotdb-*-bin.zip \ - && mv /apache-iotdb-* /iotdb \ - && apt remove unzip -y \ - && apt autoremove -y \ - && apt purge --auto-remove -y \ - && apt clean -y -EXPOSE 6667 -EXPOSE 31999 -EXPOSE 5555 -EXPOSE 8181 -VOLUME /iotdb/data -VOLUME /iotdb/logs -ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}" -ENTRYPOINT ["/iotdb/sbin/start-server.sh"] +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=3nodes_iotdb-server_1:9003 +default_replica_num=1 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false diff --git a/test/e2e/base/docker-compose.yaml b/testcontainer/src/test/resources/3nodes/docker-compose.yaml similarity index 68% copy from test/e2e/base/docker-compose.yaml copy to testcontainer/src/test/resources/3nodes/docker-compose.yaml index d4970d4..bbb18cc 100644 --- a/test/e2e/base/docker-compose.yaml +++ b/testcontainer/src/test/resources/3nodes/docker-compose.yaml @@ -20,32 +20,24 @@ version: '3.8' services: - server-prototype: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - ports: - - 6667:6667 + iotdb-server: + image: apache/iotdb:cluster-maven-development + expose: + - 6667 + - 9003 + - 40010 networks: - iotdb: + - iotdb healthcheck: test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] interval: 5s timeout: 60s retries: 120 + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 3 - initializer: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - networks: - iotdb: - entrypoint: - - bash - - -c - - | - cat /res/init.sql | grep -v '^--' | xargs -I {} /iotdb/sbin/start-cli.sh -h server -e {} - echo "Ready to Run IoTDB E2E Tests" networks: iotdb: + driver: bridge diff --git a/docker/src/main/Dockerfile b/testcontainer/src/test/resources/3nodes/iotdb-cluster.properties similarity index 57% copy from docker/src/main/Dockerfile copy to testcontainer/src/test/resources/3nodes/iotdb-cluster.properties index 402227c..fe6133d 100644 --- a/docker/src/main/Dockerfile +++ b/testcontainer/src/test/resources/3nodes/iotdb-cluster.properties @@ -15,29 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -# docker build context is the root path of the repository - -FROM openjdk:11-jre-slim - -ADD distribution/target/apache-iotdb-*-all-bin.zip / - -RUN apt update \ - && apt install lsof procps unzip -y \ - && unzip /apache-iotdb-*-bin.zip -d / \ - && rm /apache-iotdb-*-bin.zip \ - && mv /apache-iotdb-* /iotdb \ - && apt remove unzip -y \ - && apt autoremove -y \ - && apt purge --auto-remove -y \ - && apt clean -y -EXPOSE 6667 -EXPOSE 31999 -EXPOSE 5555 -EXPOSE 8181 -VOLUME /iotdb/data -VOLUME /iotdb/logs -ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}" -ENTRYPOINT ["/iotdb/sbin/start-server.sh"] +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=3nodes_iotdb-server_1:9003,3nodes_iotdb-server_2:9003,3nodes_iotdb-server_3:9003 +default_replica_num=3 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false diff --git a/test/e2e/base/docker-compose.yaml b/testcontainer/src/test/resources/5nodes/docker-compose.yaml similarity index 68% copy from test/e2e/base/docker-compose.yaml copy to testcontainer/src/test/resources/5nodes/docker-compose.yaml index d4970d4..d45577f 100644 --- a/test/e2e/base/docker-compose.yaml +++ b/testcontainer/src/test/resources/5nodes/docker-compose.yaml @@ -20,32 +20,23 @@ version: '3.8' services: - server-prototype: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - ports: - - 6667:6667 + iotdb-server: + image: apache/iotdb:cluster-maven-development + expose: + - 6667 + - 9003 + - 40010 networks: - iotdb: + - iotdb healthcheck: test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] interval: 5s timeout: 60s retries: 120 - - initializer: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile - networks: - iotdb: - entrypoint: - - bash - - -c - - | - cat /res/init.sql | grep -v '^--' | xargs -I {} /iotdb/sbin/start-cli.sh -h server -e {} - echo "Ready to Run IoTDB E2E Tests" + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 5 networks: iotdb: + driver: bridge diff --git a/docker/src/main/Dockerfile b/testcontainer/src/test/resources/5nodes/iotdb-cluster.properties similarity index 57% rename from docker/src/main/Dockerfile rename to testcontainer/src/test/resources/5nodes/iotdb-cluster.properties index 402227c..705cca6 100644 --- a/docker/src/main/Dockerfile +++ b/testcontainer/src/test/resources/5nodes/iotdb-cluster.properties @@ -15,29 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -# docker build context is the root path of the repository - -FROM openjdk:11-jre-slim - -ADD distribution/target/apache-iotdb-*-all-bin.zip / - -RUN apt update \ - && apt install lsof procps unzip -y \ - && unzip /apache-iotdb-*-bin.zip -d / \ - && rm /apache-iotdb-*-bin.zip \ - && mv /apache-iotdb-* /iotdb \ - && apt remove unzip -y \ - && apt autoremove -y \ - && apt purge --auto-remove -y \ - && apt clean -y -EXPOSE 6667 -EXPOSE 31999 -EXPOSE 5555 -EXPOSE 8181 -VOLUME /iotdb/data -VOLUME /iotdb/logs -ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}" -ENTRYPOINT ["/iotdb/sbin/start-server.sh"] +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=5nodes_iotdb-server_1:9003,5nodes_iotdb-server_2:9003,5nodes_iotdb-server_3:9003,5nodes_iotdb-server_4:9003,5nodes_iotdb-server_5:9003 +default_replica_num=3 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false diff --git a/testcontainer/src/test/resources/iotdb-engine.properties b/testcontainer/src/test/resources/iotdb-engine.properties new file mode 100644 index 0000000..960aa13 --- /dev/null +++ b/testcontainer/src/test/resources/iotdb-engine.properties @@ -0,0 +1,24 @@ +#Licensed to the Apache Software Foundation (ASF) under one +#or more contributor license agreements. See the NOTICE file +#distributed with this work for additional information +#regarding copyright ownership. The ASF licenses this file +#to you under the Apache License, Version 2.0 (the +#"License"); you may not use this file except in compliance +#with the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, +#software distributed under the License is distributed on an +#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +#KIND, either express or implied. See the License for the +#specific language governing permissions and limitations +#under the License. + + +base_dir=target/tmp +data_dirs=target/data +wal_dir=target/wal +index_root_dir=target/index +udf_root_dir=target/ext +tracing_dir=target/data/tracing \ No newline at end of file
