This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch CrossTest in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d2e0aaed66060463cfa6800df4154884e6716791 Author: JackieTien97 <[email protected]> AuthorDate: Mon Oct 31 19:25:55 2022 +0800 Move IT cross-test module into integration-test --- cross-tests/pom.xml | 125 ------ .../src/test/resources/iotdb-engine.properties | 21 - integration-test/import-control.xml | 11 + integration-test/pom.xml | 11 + .../org/apache/iotdb/tool/it}/AbstractScript.java | 18 +- .../org/apache/iotdb/tool/it}/ExportCsvTestIT.java | 83 ++-- .../apache/iotdb/tool/it}/ExportTsFileTestIT.java | 28 +- .../org/apache/iotdb/tool/it}/ImportCsvTestIT.java | 148 +++---- pom.xml | 1 - test/e2e/base/docker-compose.yaml | 51 --- test/e2e/cases/README.md | 53 --- test/e2e/cases/cli/README.md | 24 -- test/e2e/cases/cli/cleanup.sh | 27 -- test/e2e/cases/cli/docker-compose.yaml | 41 -- test/e2e/cases/cli/res/init.sql | 26 -- test/e2e/cases/cli/run.sh | 48 --- testcontainer/Readme.md | 62 --- .../java/org/apache/iotdb/db/sql/ClusterIT.java | 105 ----- .../iotdb/db/sql/ClusterSessionSimpleIT.java | 85 ---- .../java/org/apache/iotdb/db/sql/SingleNodeIT.java | 88 ----- .../iotdb/db/sql/node1/OneNodeClusterIT.java | 54 --- .../db/sql/nodes3/AbstractThreeNodeClusterIT.java | 61 --- .../iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java | 23 -- .../iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java | 32 -- .../db/sql/nodes5/AbstractFiveNodeClusterIT.java | 85 ---- .../iotdb/db/sql/nodes5/FiveNodeCluster1IT.java | 23 -- .../iotdb/db/sql/nodes5/FiveNodeCluster2IT.java | 32 -- .../iotdb/db/sql/nodes5/FiveNodeCluster4IT.java | 32 -- .../test/java/org/apache/iotdb/db/sync/SyncIT.java | 426 --------------------- .../apache/iotdb/db/sync/SyncWeakNetworkIT.java | 40 -- .../NoProjectNameDockerComposeContainer.java | 38 -- .../src/test/resources/logback-container.xml | 39 -- testcontainer/src/test/resources/logback.xml | 52 --- .../src/test/resources/sync/docker-compose.yaml | 52 --- testcontainer/src/tool/README.md | 33 -- testcontainer/src/tool/parser.py | 79 ---- 36 files changed, 140 insertions(+), 2017 deletions(-) diff --git a/cross-tests/pom.xml b/cross-tests/pom.xml deleted file mode 100644 index d65af955c1..0000000000 --- a/cross-tests/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>iotdb-parent</artifactId> - <groupId>org.apache.iotdb</groupId> - <version>0.14.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>cross-tests</artifactId> - <properties> - <cross.test.test.skip>false</cross.test.test.skip> - <cross.test.it.skip>${cross.test.test.skip}</cross.test.it.skip> - <cross.ut.skip>${cross.test.test.skip}</cross.ut.skip> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.iotdb</groupId> - <artifactId>iotdb-cli</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.iotdb</groupId> - <artifactId>iotdb-server</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.iotdb</groupId> - <artifactId>iotdb-server</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-csv</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <!--using `mvn test` to run UT, `mvn verify` to run ITs - Reference: https://antoniogoncalves.org/2012/12/13/lets-turn-integration-tests-with-maven-to-a-first-class-citizen/--> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <skipTests>${cross.ut.skip}</skipTests> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <executions> - <execution> - <id>run-integration-tests</id> - <phase>integration-test</phase> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - <configuration> - <skipTests>${cross.test.test.skip}</skipTests> - <skipITs>${cross.test.it.skip}</skipITs> - </configuration> - </plugin> - </plugins> - </build> - <profiles> - <profile> - <id>skipCrossTests</id> - <activation> - <property> - <name>skipTests</name> - <value>true</value> - </property> - </activation> - <properties> - <cross.test.it.skip>true</cross.test.it.skip> - <cross.ut.skip>true</cross.ut.skip> - <cross.test.test.skip>true</cross.test.test.skip> - </properties> - </profile> - <profile> - <id>skipUT_CrossTests</id> - <activation> - <property> - <name>skipUTs</name> - <value>true</value> - </property> - </activation> - <properties> - <cross.ut.skip>true</cross.ut.skip> - </properties> - </profile> - </profiles> -</project> diff --git a/cross-tests/src/test/resources/iotdb-engine.properties b/cross-tests/src/test/resources/iotdb-engine.properties deleted file mode 100644 index b213e4cad9..0000000000 --- a/cross-tests/src/test/resources/iotdb-engine.properties +++ /dev/null @@ -1,21 +0,0 @@ -#Licensed to the Apache Software Foundation (ASF) under one -#or more contributor license agreements. See the NOTICE file -#distributed with this work for additional information -#regarding copyright ownership. The ASF licenses this file -#to you under the Apache License, Version 2.0 (the -#"License"); you may not use this file except in compliance -#with the License. You may obtain a copy of the License at -# -#http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, -#software distributed under the License is distributed on an -#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -#KIND, either express or implied. See the License for the -#specific language governing permissions and limitations -#under the License. - - -base_dir=target/tmp -data_dirs=target/data -wal_dir=target/wal \ No newline at end of file diff --git a/integration-test/import-control.xml b/integration-test/import-control.xml index f801fafdc8..80104d913d 100644 --- a/integration-test/import-control.xml +++ b/integration-test/import-control.xml @@ -95,4 +95,15 @@ <allow pkg="org.apache.iotdb.db.metadata.idtable.trigger_example" /> <allow pkg="org.apache.iotdb.session.template" /> </subpackage> + <subpackage name="tool.it"> + <allow class="org.apache.commons.csv.CSVParser" /> + <allow class="org.apache.commons.csv.CSVRecord" /> + <allow class="org.apache.iotdb.session.ISession" /> + <allow class="org.apache.iotdb.tsfile.read.TsFileReader" /> + <allow class="org.apache.iotdb.tsfile.read.TsFileSequenceReader" /> + <allow class="org.apache.iotdb.tsfile.read.common.Path" /> + <allow class="org.apache.iotdb.tsfile.read.common.RowRecord" /> + <allow class="org.apache.iotdb.tsfile.read.expression.QueryExpression" /> + <allow class="org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet" /> + </subpackage> </import-control> diff --git a/integration-test/pom.xml b/integration-test/pom.xml index e5d8b9fe1a..2a23456af0 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -77,6 +77,17 @@ <artifactId>rewrite-tsfile-tool</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-cli</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-csv</artifactId> + <scope>test</scope> + </dependency> </dependencies> <dependencyManagement> <dependencies> diff --git a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/AbstractScript.java b/integration-test/src/test/java/org/apache/iotdb/tool/it/AbstractScript.java similarity index 89% rename from cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/AbstractScript.java rename to integration-test/src/test/java/org/apache/iotdb/tool/it/AbstractScript.java index c90cf0691c..c9cb3dc25e 100644 --- a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/AbstractScript.java +++ b/integration-test/src/test/java/org/apache/iotdb/tool/it/AbstractScript.java @@ -16,12 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cross.tests.tools.importCsv; +package org.apache.iotdb.tool.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.lang3.ArrayUtils; import org.apache.thrift.annotation.Nullable; +import org.junit.runner.RunWith; import java.io.BufferedReader; import java.io.File; @@ -35,6 +39,7 @@ import java.util.Properties; import static org.junit.Assert.assertTrue; +@RunWith(IoTDBTestRunner.class) public abstract class AbstractScript { protected String[] command; protected final String CSV_FILE = "target" + File.separator + "test.csv"; @@ -105,7 +110,16 @@ public abstract class AbstractScript { protected void testMethod(@Nullable String[] params, @Nullable String[] output) throws IOException { String[] basicParams = - new String[] {"-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root"}; + new String[] { + "-h", + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getIp(), + "-p", + String.valueOf(EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPort()), + "-u", + "root", + "-pw", + "root" + }; command = ArrayUtils.addAll(command, basicParams); command = ArrayUtils.addAll(command, params); if (params != null) { diff --git a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ExportCsvTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tool/it/ExportCsvTestIT.java similarity index 62% rename from cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ExportCsvTestIT.java rename to integration-test/src/test/java/org/apache/iotdb/tool/it/ExportCsvTestIT.java index a492dd5505..6eaef41195 100644 --- a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ExportCsvTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tool/it/ExportCsvTestIT.java @@ -17,12 +17,14 @@ * under the License. */ -package org.apache.iotdb.cross.tests.tools.importCsv; +package org.apache.iotdb.tool.it; -import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.ISession; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -30,6 +32,8 @@ import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; import java.io.File; import java.io.IOException; @@ -38,11 +42,13 @@ import java.util.List; import static org.junit.Assert.assertEquals; +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) public class ExportCsvTestIT extends AbstractScript { @Before - public void setUp() { - EnvironmentUtils.envSetUp(); + public void setUp() throws Exception { + EnvFactory.getEnv().initBeforeTest(); String os = System.getProperty("os.name").toLowerCase(); if (os.startsWith("windows")) { command = @@ -61,7 +67,7 @@ public class ExportCsvTestIT extends AbstractScript { @After public void tearDown() throws Exception { - EnvironmentUtils.cleanEnv(); + EnvFactory.getEnv().cleanAfterTest(); } @Test @@ -70,15 +76,16 @@ public class ExportCsvTestIT extends AbstractScript { String[] params = {"-td", "target/", "-q", "select c1,c2,c3 from root.test.t1"}; prepareData(); testMethod(params, null); - CSVParser parser = readCsvFile("target/dump0_0.csv"); - String[] realRecords = { - "root.test.t1.c1,root.test.t1.c2,root.test.t1.c3", "1.0,\"\"abc\",aa\",\"abbe's\"" - }; - List<CSVRecord> records = parser.getRecords(); - for (int i = 0; i < records.size(); i++) { - String record = StringUtils.join(records.get(i).toList(), ','); - record = record.substring(record.indexOf(',') + 1); - assertEquals(realRecords[i], record); + try (CSVParser parser = readCsvFile("target/dump0_0.csv")) { + String[] realRecords = { + "root.test.t1.c1,root.test.t1.c2,root.test.t1.c3", "1.0,\"\"abc\",aa\",\"abbe's\"" + }; + List<CSVRecord> records = parser.getRecords(); + for (int i = 0; i < records.size(); i++) { + String record = StringUtils.join(records.get(i).toList(), ','); + record = record.substring(record.indexOf(',') + 1); + assertEquals(realRecords[i], record); + } } } @@ -90,16 +97,17 @@ public class ExportCsvTestIT extends AbstractScript { }; prepareData(); testMethod(params, null); - CSVParser parser = readCsvFile("target/dump0_0.csv"); - String[] realRecords = { - "root.test.t1.c1(FLOAT),root.test.t1.c2(TEXT),root.test.t1.c3(TEXT)", - "1.0,\"\"abc\",aa\",\"abbe's\"" - }; - List<CSVRecord> records = parser.getRecords(); - for (int i = 0; i < records.size(); i++) { - String record = StringUtils.join(records.get(i).toList(), ','); - record = record.substring(record.indexOf(',') + 1); - assertEquals(realRecords[i], record); + try (CSVParser parser = readCsvFile("target/dump0_0.csv")) { + String[] realRecords = { + "root.test.t1.c1(FLOAT),root.test.t1.c2(TEXT),root.test.t1.c3(TEXT)", + "1.0,\"\"abc\",aa\",\"abbe's\"" + }; + List<CSVRecord> records = parser.getRecords(); + for (int i = 0; i < records.size(); i++) { + String record = StringUtils.join(records.get(i).toList(), ','); + record = record.substring(record.indexOf(',') + 1); + assertEquals(realRecords[i], record); + } } } @@ -111,21 +119,20 @@ public class ExportCsvTestIT extends AbstractScript { }; prepareData(); testMethod(params, null); - CSVParser parser = readCsvFile("target/dump0_0.csv"); - String[] realRecords = { - "count(root.test.t1.c1),count(root.test.t1.c2),count(root.test.t1.c3)", "1,1,1" - }; - List<CSVRecord> records = parser.getRecords(); - for (int i = 0; i < records.size(); i++) { - String record = StringUtils.join(records.get(i).toList(), ','); - assertEquals(realRecords[i], record); + try (CSVParser parser = readCsvFile("target/dump0_0.csv")) { + String[] realRecords = { + "count(root.test.t1.c1),count(root.test.t1.c2),count(root.test.t1.c3)", "1,1,1" + }; + List<CSVRecord> records = parser.getRecords(); + for (int i = 0; i < records.size(); i++) { + String record = StringUtils.join(records.get(i).toList(), ','); + assertEquals(realRecords[i], record); + } } } private void prepareData() throws IoTDBConnectionException, StatementExecutionException { - Session session = null; - try { - session = new Session("127.0.0.1", 6667, "root", "root"); + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { session.open(); String deviceId = "root.test.t1"; @@ -139,10 +146,6 @@ public class ExportCsvTestIT extends AbstractScript { values.add("\"abc\",aa"); values.add("abbe's"); session.insertRecord(deviceId, 1L, measurements, values); - } finally { - if (session != null) { - session.close(); - } } } } diff --git a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/tsfile/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tool/it/ExportTsFileTestIT.java similarity index 86% rename from cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/tsfile/ExportTsFileTestIT.java rename to integration-test/src/test/java/org/apache/iotdb/tool/it/ExportTsFileTestIT.java index 9370af45af..9b4b166514 100644 --- a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/tsfile/ExportTsFileTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tool/it/ExportTsFileTestIT.java @@ -17,13 +17,14 @@ * under the License. */ -package org.apache.iotdb.cross.tests.tools.tsfile; +package org.apache.iotdb.tool.it; -import org.apache.iotdb.cross.tests.tools.importCsv.AbstractScript; -import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.ISession; import org.apache.iotdb.tsfile.read.TsFileReader; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; @@ -35,6 +36,8 @@ import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; import java.io.File; import java.io.IOException; @@ -43,11 +46,13 @@ import java.util.List; import static org.junit.Assert.assertEquals; +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) public class ExportTsFileTestIT extends AbstractScript { @Before - public void setUp() { - EnvironmentUtils.envSetUp(); + public void setUp() throws Exception { + EnvFactory.getEnv().initBeforeTest(); String os = System.getProperty("os.name").toLowerCase(); if (os.startsWith("windows")) { command = @@ -66,7 +71,7 @@ public class ExportTsFileTestIT extends AbstractScript { @After public void tearDown() throws Exception { - EnvironmentUtils.cleanEnv(); + EnvFactory.getEnv().cleanAfterTest(); } @Test @@ -98,9 +103,7 @@ public class ExportTsFileTestIT extends AbstractScript { } private void prepareData() throws IoTDBConnectionException, StatementExecutionException { - Session session = null; - try { - session = new Session("127.0.0.1", 6667, "root", "root"); + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { session.open(); String deviceId = "root.test.t2"; @@ -114,11 +117,6 @@ public class ExportTsFileTestIT extends AbstractScript { values.add("bbbbb"); values.add("abbes"); session.insertRecord(deviceId, 1L, measurements, values); - - } finally { - if (session != null) { - session.close(); - } } } } diff --git a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ImportCsvTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tool/it/ImportCsvTestIT.java similarity index 69% rename from cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ImportCsvTestIT.java rename to integration-test/src/test/java/org/apache/iotdb/tool/it/ImportCsvTestIT.java index 13fb2e0315..72709e9fe5 100644 --- a/cross-tests/src/test/java/org/apache/iotdb/cross/tests/tools/importCsv/ImportCsvTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tool/it/ImportCsvTestIT.java @@ -17,23 +17,25 @@ * under the License. */ -package org.apache.iotdb.cross.tests.tools.importCsv; +package org.apache.iotdb.tool.it; -import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.jdbc.Config; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -43,20 +45,10 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) public class ImportCsvTestIT extends AbstractScript { - private static final String[] sqls = - new String[] { - "SET STORAGE GROUP TO root.fit.d1", - "SET STORAGE GROUP TO root.fit.d2", - "SET STORAGE GROUP TO root.fit.p", - "CREATE TIMESERIES root.fit.d1.s1 WITH DATATYPE=INT32,ENCODING=RLE", - "CREATE TIMESERIES root.fit.d1.s2 WITH DATATYPE=TEXT,ENCODING=PLAIN", - "CREATE TIMESERIES root.fit.d2.s1 WITH DATATYPE=INT32,ENCODING=RLE", - "CREATE TIMESERIES root.fit.d2.s3 WITH DATATYPE=INT32,ENCODING=RLE", - "CREATE TIMESERIES root.fit.p.s1 WITH DATATYPE=INT32,ENCODING=RLE", - }; - private final String[] noDataOutput = { "````````````````````````````````````````````````", "Starting IoTDB Client Import Script", @@ -80,9 +72,9 @@ public class ImportCsvTestIT extends AbstractScript { }; @Before - public void setUp() { + public void setUp() throws Exception { // start an IotDB server environment - EnvironmentUtils.envSetUp(); + EnvFactory.getEnv().initBeforeTest(); // choose an execute command by system. String os = System.getProperty("os.name").toLowerCase(); if (os.startsWith("windows")) { @@ -103,28 +95,12 @@ public class ImportCsvTestIT extends AbstractScript { @After public void tearDown() throws Exception { // shutdown IotDB server environment - EnvironmentUtils.cleanEnv(); - } - - private static void createSchema() throws ClassNotFoundException { - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - - for (String sql : sqls) { - statement.execute(sql); - } - } catch (Exception e) { - e.printStackTrace(); - } + EnvFactory.getEnv().cleanAfterTest(); } - /** * test the situation that the schema has not been created and CSV file has no records * - * @throws IOException + * @throws java.io.IOException */ @Test public void testImportNoRecordsCSV() throws IOException { @@ -140,7 +116,7 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the CSV file has no headers * - * @throws IOException + * @throws java.io.IOException */ @Test public void testNoHeader() throws IOException { @@ -152,7 +128,7 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the CSV file is an empty file * - * @throws IOException + * @throws java.io.IOException */ @Test public void testEmptyCSV() throws IOException { @@ -164,24 +140,18 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the schema has been created and CSV file has no problem * - * @throws IOException + * @throws java.io.IOException */ @Test - public void test() throws IOException, ClassNotFoundException { + public void test() throws IOException { assertTrue(generateTestCSV(false, false, false, false, false)); String[] params = {"-f", CSV_FILE}; testMethod(params, null); File file = new File(CSV_FILE); - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - if (statement.execute("select * from root.**")) { - ResultSet resultSet = statement.getResultSet(); - testResult(resultSet, 6, 3); - resultSet.close(); - } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from root.**")) { + testResult(resultSet, 6, 3); } catch (Exception e) { e.printStackTrace(); } @@ -193,25 +163,19 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the schema has been created and CSV file has no problem * - * @throws IOException + * @throws java.io.IOException */ @Test - public void testAligned() throws IOException, ClassNotFoundException { + public void testAligned() throws IOException { assertTrue(generateTestCSV(false, false, false, false, false)); String[] params = {"-f", CSV_FILE, "-aligned "}; testMethod(params, null); File file = new File(CSV_FILE); - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - if (statement.execute("show devices")) { - ResultSet resultSet = statement.getResultSet(); - while (resultSet.next()) { - assertTrue("true".equals(resultSet.getString(2))); - } - resultSet.close(); + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("show devices")) { + while (resultSet.next()) { + assertEquals("true", resultSet.getString(2)); } } catch (Exception e) { e.printStackTrace(); @@ -224,24 +188,18 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the schema has not been created and CSV file has no problem * - * @throws IOException + * @throws java.io.IOException */ @Test - public void testWithoutCreateSchema() throws IOException, ClassNotFoundException { + public void testWithoutCreateSchema() throws IOException { assertTrue(generateTestCSV(false, false, false, false, false)); String[] params = {"-f", CSV_FILE}; testMethod(params, null); File file = new File(CSV_FILE); - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - if (statement.execute("select * from root.**")) { - ResultSet resultSet = statement.getResultSet(); - testResult(resultSet, 6, 3); - resultSet.close(); - } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from root.**")) { + testResult(resultSet, 6, 3); } catch (Exception e) { e.printStackTrace(); } @@ -253,25 +211,18 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the schema has not been created and CSV file has no data type * - * @throws IOException - * @throws ClassNotFoundException + * @throws java.io.IOException */ @Test - public void testWithDataType() throws IOException, ClassNotFoundException { + public void testWithDataType() throws IOException { assertTrue(generateTestCSV(false, false, false, false, true)); String[] params = {"-f", CSV_FILE}; testMethod(params, null); File file = new File(CSV_FILE); - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - if (statement.execute("select * from root.**")) { - ResultSet resultSet = statement.getResultSet(); - testResult(resultSet, 6, 3); - resultSet.close(); - } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from root.**")) { + testResult(resultSet, 6, 3); } catch (Exception e) { e.printStackTrace(); } @@ -283,25 +234,18 @@ public class ImportCsvTestIT extends AbstractScript { /** * test the situation that the schema has not been created and CSV file has no data type * - * @throws IOException - * @throws ClassNotFoundException + * @throws java.io.IOException */ @Test - public void testWithException() throws IOException, ClassNotFoundException { + public void testWithException() throws IOException { assertTrue(generateTestCSV(false, false, false, true, true)); String[] params = {"-f", CSV_FILE}; testMethod(params, null); File file = new File(CSV_FILE); - Class.forName(Config.JDBC_DRIVER_NAME); - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - if (statement.execute("select ** from root")) { - ResultSet resultSet = statement.getResultSet(); - testResult(resultSet, 6, 3); - resultSet.close(); - } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select ** from root")) { + testResult(resultSet, 6, 3); } catch (Exception e) { e.printStackTrace(); } @@ -323,7 +267,7 @@ public class ImportCsvTestIT extends AbstractScript { /** * test whether the shape of data is correct * - * @throws IOException + * @throws java.io.IOException */ private static void testResult( ResultSet resultSet, int expectedColumnNumber, int expectedRowNumber) throws SQLException { diff --git a/pom.xml b/pom.xml index 192c89a951..50f6fa64c2 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,6 @@ <module>hive-connector</module> <module>node-commons</module> <module>confignode</module> - <module>cross-tests</module> <module>zeppelin-interpreter</module> <module>client-py</module> <module>compile-tools</module> diff --git a/test/e2e/base/docker-compose.yaml b/test/e2e/base/docker-compose.yaml index f536ec6966..e69de29bb2 100644 --- a/test/e2e/base/docker-compose.yaml +++ b/test/e2e/base/docker-compose.yaml @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -version: '3.8' - -services: - server-prototype: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile-single - ports: - - 6667:6667 - networks: - iotdb: - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 - - initializer: - build: - context: ../../.. - dockerfile: docker/src/main/Dockerfile-single - networks: - iotdb: - entrypoint: - - bash - - -c - - | - cat /res/init.sql | grep -v '^--' | xargs -I {} /iotdb/sbin/start-cli.sh -h server -e {} - echo "Ready to Run IoTDB E2E Tests" - -networks: - iotdb: diff --git a/test/e2e/cases/README.md b/test/e2e/cases/README.md index c4a15e3b96..e69de29bb2 100644 --- a/test/e2e/cases/README.md +++ b/test/e2e/cases/README.md @@ -1,53 +0,0 @@ -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> - -# IoTDB E2E tests cases - -Test cases are organized into sub-directories, each of which contains the following files: - -* `run.sh`: the entry of the test case. -* `cleanup.sh`: a cleanup script to clean up resources that are created during the test. -* `res`: resources files that will be mounted into the container(s) and be used there. -* `docker-compose.yaml`: orchestrates the services used in the test process. -* `README.md` (Optional): docs or notes when running this case manually. - -any other additional files are completely acceptable here, for example, when building -a case to test the JDBC SDK, the files structure may be something like: - -```text -. -├── README.md -├── cleanup.sh -├── docker-compose.yaml -├── app <------- Java application that uses JDBC SDK to communicate with IoTDB -│ ├── pom.xml -│ ├── src -│ │ ├── main -│ │ │ └── java -│ │ └── test -│ │ └── java -│ └── src -│ ├── main -│ └── test -├── res -│ └── init.sql -└── run.sh -``` diff --git a/test/e2e/cases/cli/README.md b/test/e2e/cases/cli/README.md index 343abebc56..e69de29bb2 100644 --- a/test/e2e/cases/cli/README.md +++ b/test/e2e/cases/cli/README.md @@ -1,24 +0,0 @@ -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> - -# Standalone Server Test - -The simplest test case that starts up an IoTDB server and verifies that the CLI works. diff --git a/test/e2e/cases/cli/cleanup.sh b/test/e2e/cases/cli/cleanup.sh old mode 100755 new mode 100644 index a96f4d06c5..e69de29bb2 --- a/test/e2e/cases/cli/cleanup.sh +++ b/test/e2e/cases/cli/cleanup.sh @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x - -cd "$(dirname "$0")" || exit 1 - -docker-compose down - -cd - || exit 1 diff --git a/test/e2e/cases/cli/docker-compose.yaml b/test/e2e/cases/cli/docker-compose.yaml index 2e9da0422e..e69de29bb2 100644 --- a/test/e2e/cases/cli/docker-compose.yaml +++ b/test/e2e/cases/cli/docker-compose.yaml @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -version: '3.8' - -services: - server: - extends: - file: ../../base/docker-compose.yaml - service: server-prototype - volumes: - - ./res:/resources - - initializer: - extends: - service: initializer - file: ../../base/docker-compose.yaml - volumes: - - ./res:/res:ro - depends_on: - server: - condition: service_healthy - -networks: - iotdb: diff --git a/test/e2e/cases/cli/res/init.sql b/test/e2e/cases/cli/res/init.sql index cb9f6aade0..e69de29bb2 100644 --- a/test/e2e/cases/cli/res/init.sql +++ b/test/e2e/cases/cli/res/init.sql @@ -1,26 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. --- - -SET STORAGE GROUP TO root.ln; -SHOW STORAGE GROUP; - -CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=PLAIN; - -INSERT INTO root.ln.wf01.wt01(timestamp,temperature) values(100, 16); -INSERT INTO root.ln.wf01.wt01(timestamp,temperature) values(200, 26); diff --git a/test/e2e/cases/cli/run.sh b/test/e2e/cases/cli/run.sh old mode 100755 new mode 100644 index a62bba7ecb..e69de29bb2 --- a/test/e2e/cases/cli/run.sh +++ b/test/e2e/cases/cli/run.sh @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -cd "$(dirname "$0")" - -docker-compose up -d - -max_attempts=10 -attempts=1 - -while ! docker-compose logs | grep -c 'Ready to Run IoTDB E2E Tests' > /dev/null 2>&1; do - if [[ $attempts -gt $max_attempts ]]; then - echo "Preparation is not ready after $max_attempts attempts, will exit now" - exit 1 - fi - echo "Preparation is not ready yet, retrying ($attempts/$max_attempts)" - sleep 3 - attempts=$((attempts+1)) -done - -results=$(docker-compose exec -T server /iotdb/sbin/start-cli.sh -e 'SELECT temperature FROM root.ln.wf01.wt01') - -if [[ $results != *"Total line number = 2"* ]]; then - echo "Total line number should be 2" - echo "$results" - exit 1 -fi - -cd - diff --git a/testcontainer/Readme.md b/testcontainer/Readme.md index d4899781d4..e69de29bb2 100644 --- a/testcontainer/Readme.md +++ b/testcontainer/Readme.md @@ -1,62 +0,0 @@ -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> - -# Description - -This module is for using Docker and TestContainer for end to end test. - -## Requirements - -You have to install Docker before you runn this module. -See [Docker Version Requirement](https://www.testcontainers.org/supported_docker_environment/). - -IoTDB project will detect whether a Docker is installed (but do not check the docker's version). - -The logic is, for Unix-like system, it checks whether `/var/run/docker.sock` exists. -For Window system, it checks whether `C:\Program Files\Docker\Docker\resources\bin\docker.exe` exists. - -If you are sure you have installed the Docker but `testcontainer` module is not loaed, use `-P unixDockerCheck` -in your maven command, which also works on Windows OS. - -## Behavior - -Before running `integration-test` in this module, binaries must be generated in the `distribution` module, -e.g, call `mvn package -Dmaven.test.skip=true`. - -In this module, when running `mvn pre-integration-test` (or `mvn integration-test`, `mvn post-integration-test`), -the module will build docker image, `apache/iotdb:maven-development`. - -In the `post-integration-test` phase, the above images will be removed. - -In the `integration-test` phase, all `src/test/java/**/*IT.java` will be tested. - -## How it runs - -`apache/iotdb:maven-development` is generated following the Dockerfile `${basedir}/docker/src/main/Dockerfile-single`, and - -For testing sync module only, we use `mvn integration-test -P sync`. - -TestContainer can start the docker (or docker compose) automatically. - -But these docker compose files can also be used independently. -e.g., `docker-compose up`. - - diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java index 71d8882608..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterIT.java @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql; - -import org.apache.iotdb.jdbc.Config; -import org.apache.iotdb.session.Session; - -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; -import java.util.concurrent.TimeUnit; - -// do not add tests here. -// add tests into Cases.java instead. -public abstract class ClusterIT extends Cases { - - private static Logger logger = LoggerFactory.getLogger(ClusterIT.class); - - // "root.sg1" is a special storage for testing whether the read and write operations can be run - // correctly if the data is not on the connected node. - public String defaultSG = "root.sg1"; - - protected int getWriteRpcPort() { - return getContainer().getServicePort("iotdb-server_1", 6667); - } - - protected String getWriteRpcIp() { - return getContainer().getServiceHost("iotdb-server_1", 6667); - } - - protected int[] getReadRpcPorts() { - return new int[] {getContainer().getServicePort("iotdb-server_1", 6667)}; - } - - protected String[] getReadRpcIps() { - return new String[] {getContainer().getServiceHost("iotdb-server_1", 6667)}; - } - - protected void startCluster() {} - - protected abstract DockerComposeContainer getContainer(); - - @Before - public void init() throws Exception { - startCluster(); - - Class.forName(Config.JDBC_DRIVER_NAME); - writeConnection = - DriverManager.getConnection( - "jdbc:iotdb://" + getWriteRpcIp() + ":" + getWriteRpcPort(), "root", "root"); - writeStatement = writeConnection.createStatement(); - - int[] readPorts = getReadRpcPorts(); - String[] readIps = getReadRpcIps(); - readConnections = new Connection[readPorts.length]; - readStatements = new Statement[readPorts.length]; - for (int i = 0; i < readPorts.length; i++) { - readConnections[i] = - DriverManager.getConnection( - "jdbc:iotdb://" + readIps[i] + ":" + readPorts[i], "root", "root"); - readStatements[i] = readConnections[i].createStatement(); - } - session = - new Session.Builder() - .host(getWriteRpcIp()) - .port(getWriteRpcPort()) - .username("root") - .password("root") - .enableRedirection(false) - .build(); - session.open(); - TimeUnit.MILLISECONDS.sleep(3000); - } - - @After - public void clean() throws Exception { - super.clean(); - } - - // do not add tests here. - // add tests into Cases.java instead. -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterSessionSimpleIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterSessionSimpleIT.java index 6f9189fd5e..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterSessionSimpleIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterSessionSimpleIT.java @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql; - -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; - -import org.junit.Rule; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.NoProjectNameDockerComposeContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -public class ClusterSessionSimpleIT { - - private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1"); - private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2"); - private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3"); - - private Session session; - - @Rule - public DockerComposeContainer environment = - new NoProjectNameDockerComposeContainer( - "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml")) - .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger)) - .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_2", new Slf4jLogConsumer(node2Logger)) - .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger)) - .withLocalCompose(true); - - protected DockerComposeContainer getContainer() { - return environment; - } - - @Test - public void testSessionCluster() throws IoTDBConnectionException, StatementExecutionException { - List<String> stringList = new ArrayList<>(); - Integer service1Port = getContainer().getServicePort("iotdb-server_1", 6667); - Integer service2Port = getContainer().getServicePort("iotdb-server_2", 6667); - Integer service3Port = getContainer().getServicePort("iotdb-server_3", 6667); - stringList.add("localhost:" + service1Port); - stringList.add("localhost:" + service2Port); - stringList.add("localhost:" + service3Port); - session = new Session(stringList, "root", "root"); - session.open(); - session.setStorageGroup("root.sg1"); - session.createTimeseries( - "root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - - session.createTimeseries( - "root.sg1.d2.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); - session.close(); - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeIT.java index fa75d1e63c..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeIT.java @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.sql; - -import org.apache.iotdb.jdbc.Config; -import org.apache.iotdb.session.Session; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.PullPolicy; -import org.testcontainers.utility.DockerImageName; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; - -// do not add tests here. -// add tests into Cases.java instead. -public class SingleNodeIT extends Cases { - private static Logger logger = LoggerFactory.getLogger(SingleNodeIT.class); - - @Rule - public GenericContainer dslContainer = - new GenericContainer(DockerImageName.parse("apache/iotdb:maven-development")) - .withImagePullPolicy(PullPolicy.defaultPolicy()) - // mount another properties for changing parameters, e.g., open 5555 port (sync module) - .withFileSystemBind( - new File("src/test/resources/iotdb-datanode.properties").getAbsolutePath(), - "/iotdb/conf/iotdb-datanode.properties", - BindMode.READ_ONLY) - .withFileSystemBind( - new File("src/test/resources/logback-container.xml").getAbsolutePath(), - "/iotdb/conf/logback.xml", - BindMode.READ_ONLY) - .withLogConsumer(new Slf4jLogConsumer(logger)) - .withExposedPorts(6667) - .waitingFor(Wait.forListeningPort()); - - int rpcPort = 6667; - int syncPort = 5555; - - @Before - public void init() throws Exception { - rpcPort = dslContainer.getMappedPort(6667); - syncPort = dslContainer.getMappedPort(5555); - Class.forName(Config.JDBC_DRIVER_NAME); - readConnections = new Connection[1]; - readStatements = new Statement[1]; - writeConnection = - readConnections[0] = - DriverManager.getConnection("jdbc:iotdb://127.0.0.1:" + rpcPort, "root", "root"); - writeStatement = readStatements[0] = writeConnection.createStatement(); - session = new Session("127.0.0.1", rpcPort); - session.open(); - } - - @After - public void clean() throws Exception { - super.clean(); - } - - // do not add tests here. - // add tests into Cases.java instead. -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/node1/OneNodeClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/node1/OneNodeClusterIT.java deleted file mode 100644 index daa7161ac1..0000000000 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/node1/OneNodeClusterIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.node1; - -import org.apache.iotdb.db.sql.ClusterIT; - -import org.junit.Rule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.NoProjectNameDockerComposeContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; - -// a cluster with only one node. -// in this case, the read and write connection must be on the same node, -// the data is also -public class OneNodeClusterIT extends ClusterIT { - private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1"); - - // in TestContainer's document, it is @ClassRule, and the environment is `public static` - // I am not sure the difference now. - @Rule - public DockerComposeContainer environment = - new NoProjectNameDockerComposeContainer( - "1node", new File("src/test/resources/1node/docker-compose.yaml")) - .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger)) - .withLocalCompose(true); - - @Override - protected DockerComposeContainer getContainer() { - return environment; - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java index 2729581ff2..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/AbstractThreeNodeClusterIT.java @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes3; - -import org.apache.iotdb.db.sql.ClusterIT; - -import org.junit.Rule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.NoProjectNameDockerComposeContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; - -// just add real ITs into AbstractClusterIT. -// in this case, the data must be on all nodes. -// we just simulate write data on node A and read data on either node A or B. -public abstract class AbstractThreeNodeClusterIT extends ClusterIT { - - private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1"); - private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2"); - private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3"); - - // in TestContainer's document, it is @ClassRule, and the environment is `public static` - // I am not sure the difference now. - @Rule - public DockerComposeContainer environment = - new NoProjectNameDockerComposeContainer( - "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml")) - .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger)) - .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_2", new Slf4jLogConsumer(node2Logger)) - .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger)) - .withLocalCompose(true); - - @Override - protected DockerComposeContainer getContainer() { - return environment; - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java index 4802100fbd..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster1IT.java @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes3; - -// read and the write statements are on the same node. -public class ThreeNodeCluster1IT extends AbstractThreeNodeClusterIT {} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java index 69ee3a57bf..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes3/ThreeNodeCluster2IT.java @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes3; - -// read and the write statements are on the different nodes. -public class ThreeNodeCluster2IT extends AbstractThreeNodeClusterIT { - - protected String getWriteRpcIp() { - return getContainer().getServiceHost("iotdb-server_2", 6667); - } - - protected int getWriteRpcPort() { - return getContainer().getServicePort("iotdb-server_2", 6667); - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java index 4535fdb299..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/AbstractFiveNodeClusterIT.java @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes5; - -import org.apache.iotdb.db.sql.ClusterIT; - -import org.junit.Rule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.NoProjectNameDockerComposeContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; - -// just add real ITs into AbstractClusterIT. -// in this case, the data may be not on the node that a read client connects. -// So, we have: write data on node A and read data on either node A, B, and C. -public abstract class AbstractFiveNodeClusterIT extends ClusterIT { - - private static Logger node1Logger = LoggerFactory.getLogger("iotdb-server_1"); - private static Logger node2Logger = LoggerFactory.getLogger("iotdb-server_2"); - private static Logger node3Logger = LoggerFactory.getLogger("iotdb-server_3"); - private static Logger node4Logger = LoggerFactory.getLogger("iotdb-server_4"); - private static Logger node5Logger = LoggerFactory.getLogger("iotdb-server_5"); - - // in TestContainer's document, it is @ClassRule, and the environment is `public static` - // I am not sure the difference now. - @Rule - public DockerComposeContainer environment = - new NoProjectNameDockerComposeContainer( - "5nodes", new File("src/test/resources/5nodes/docker-compose.yaml")) - .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_1", new Slf4jLogConsumer(node1Logger)) - .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_2", new Slf4jLogConsumer(node2Logger)) - .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_3", new Slf4jLogConsumer(node3Logger)) - .withExposedService("iotdb-server_4", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_4", new Slf4jLogConsumer(node4Logger)) - .withExposedService("iotdb-server_5", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-server_5", new Slf4jLogConsumer(node5Logger)) - .withLocalCompose(true); - - @Override - protected DockerComposeContainer getContainer() { - return environment; - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java index 328de27a48..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster1IT.java @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes5; - -// read and the write statements are on the different nodes. -public class FiveNodeCluster1IT extends AbstractFiveNodeClusterIT {} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java index 6a455bccb5..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster2IT.java @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes5; - -// read and the write statements are on the different nodes, and maybe in the same raft group. -public class FiveNodeCluster2IT extends AbstractFiveNodeClusterIT { - - protected String getWriteRpcIp() { - return getContainer().getServiceHost("iotdb-server_2", 6667); - } - - protected int getWriteRpcPort() { - return getContainer().getServicePort("iotdb-server_2", 6667); - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java index 2b8491f5ff..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/nodes5/FiveNodeCluster4IT.java @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.sql.nodes5; - -// read and the write statements are on the different nodes, and maybe in the different raft groups. -public class FiveNodeCluster4IT extends AbstractFiveNodeClusterIT { - - protected String getWriteRpcIp() { - return getContainer().getServiceHost("iotdb-server_4", 6667); - } - - protected int getWriteRpcPort() { - return getContainer().getServicePort("iotdb-server_4", 6667); - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncIT.java index c7caf1918d..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncIT.java @@ -1,426 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.sync; - -import org.apache.iotdb.jdbc.Config; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.NoProjectNameDockerComposeContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -public class SyncIT { - private static Logger receiverLogger = LoggerFactory.getLogger("iotdb-receiver_1"); - private static Logger senderLogger = LoggerFactory.getLogger("iotdb-sender_1"); - private static int RETRY_TIME = 30; - - protected Statement senderStatement; - protected Connection senderConnection; - protected Statement receiverStatement; - protected Connection receiverConnection; - - // in TestContainer's document, it is @ClassRule, and the environment is `public static` - // I am not sure the difference now. - @Rule - public DockerComposeContainer environment = - new NoProjectNameDockerComposeContainer( - "sync", new File("src/test/resources/sync/docker-compose.yaml")) - .withExposedService("iotdb-sender_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-sender_1", new Slf4jLogConsumer(senderLogger)) - .withExposedService("iotdb-receiver_1", 6667, Wait.forListeningPort()) - .withLogConsumer("iotdb-receiver_1", new Slf4jLogConsumer(receiverLogger)) - .withLocalCompose(true); - - protected int getSenderRpcPort() { - return environment.getServicePort("iotdb-sender_1", 6667); - } - - protected String getSenderIp() { - return environment.getServiceHost("iotdb-sender_1", 6667); - } - - protected int getReceiverRpcPort() { - return environment.getServicePort("iotdb-receiver_1", 6667); - } - - protected String getReceiverIp() { - return environment.getServiceHost("iotdb-receiver_1", 6667); - } - - @Before - public void init() throws Exception { - Class.forName(Config.JDBC_DRIVER_NAME); - senderConnection = - DriverManager.getConnection( - "jdbc:iotdb://" + getSenderIp() + ":" + getSenderRpcPort(), "root", "root"); - senderStatement = senderConnection.createStatement(); - receiverConnection = - DriverManager.getConnection( - "jdbc:iotdb://" + getReceiverIp() + ":" + getReceiverRpcPort(), "root", "root"); - receiverStatement = receiverConnection.createStatement(); - } - - @After - public void clean() throws Exception { - senderStatement.close(); - senderConnection.close(); - receiverStatement.close(); - receiverConnection.close(); - } - - private void prepareSchema() throws Exception { - senderStatement.execute("set storage group to root.sg1"); - senderStatement.execute("set storage group to root.sg2"); - senderStatement.execute("create timeseries root.sg1.d1.s1 with datatype=int32, encoding=PLAIN"); - senderStatement.execute("create timeseries root.sg1.d1.s2 with datatype=float, encoding=RLE"); - senderStatement.execute("create timeseries root.sg1.d1.s3 with datatype=TEXT, encoding=PLAIN"); - senderStatement.execute("create timeseries root.sg1.d2.s4 with datatype=int64, encoding=PLAIN"); - senderStatement.execute("create timeseries root.sg2.d1.s0 with datatype=int32, encoding=PLAIN"); - senderStatement.execute( - "create timeseries root.sg2.d2.s1 with datatype=boolean, encoding=PLAIN"); - } - - /* add one seq tsfile in sg1 */ - private void prepareIns1() throws Exception { - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(1, 1, 16.0, 'a')"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(2, 2, 25.16, 'b')"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(3, 3, 65.25, 'c')"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(16, 25, 100.0, 'd')"); - senderStatement.execute("insert into root.sg1.d2(timestamp, s4) values(1, 1)"); - senderStatement.execute("flush"); - } - - /* add one seq tsfile in sg1 */ - private void prepareIns2() throws Exception { - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(100, 65, 16.25, 'e')"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(65, 100, 25.0, 'f')"); - senderStatement.execute("insert into root.sg1.d2(timestamp, s4) values(200, 100)"); - senderStatement.execute("flush"); - } - - /* add one seq tsfile in sg1, one unseq tsfile in sg1, one seq tsfile in sg2 */ - private void prepareIns3() throws Exception { - senderStatement.execute("insert into root.sg2.d1(timestamp, s0) values(100, 100)"); - senderStatement.execute("insert into root.sg2.d1(timestamp, s0) values(65, 65)"); - senderStatement.execute("insert into root.sg2.d2(timestamp, s1) values(1, true)"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(25, 16, 65.16, 'g')"); - senderStatement.execute( - "insert into root.sg1.d1(timestamp, s1, s2, s3) values(200, 100, 16.65, 'h')"); - senderStatement.execute("flush"); - } - - private void prepareDel1() throws Exception { // after ins1, add 2 deletions - senderStatement.execute("delete from root.sg1.d1.s1 where time == 3"); - senderStatement.execute("delete from root.sg1.d1.s2 where time >= 1 and time <= 2"); - } - - private void prepareDel2() throws Exception { // after ins2, add 3 deletions - senderStatement.execute("delete from root.sg1.d1.s3 where time <= 65"); - } - - private void prepareDel3() throws Exception { // after ins3, add 5 deletions, 2 schemas{ - senderStatement.execute("delete from root.sg1.d1.* where time <= 2"); - senderStatement.execute("delete timeseries root.sg1.d2.*"); - senderStatement.execute("delete storage group root.sg2"); - } - - private void preparePipe() throws Exception { - receiverStatement.execute("start pipeserver"); - senderStatement.execute( - "create pipesink my_iotdb as iotdb(ip='sync_iotdb-receiver_1',port=6670)"); - senderStatement.execute("create pipe p to my_iotdb"); - } - - private void startPipe() throws Exception { - senderStatement.execute("start pipe p"); - } - - private void stopPipe() throws Exception { - senderStatement.execute("stop pipe p"); - } - - private void dropPipe() throws Exception { - senderStatement.execute("drop pipe p"); - } - - private void checkResult() throws Exception { - String[] columnNames = - new String[] { - "root.sg1.d1.s3", - "root.sg1.d1.s1", - "root.sg1.d1.s2", - "root.sg1.d2.s4", - "root.sg2.d1.s0", - "root.sg2.d2.s1" - }; - String[] results = - new String[] { - "1,a,1,16.0,1,null,true", - "2,b,2,25.16,null,null,null", - "3,c,3,65.25,null,null,null", - "16,d,25,100.0,null,null,null", - "25,g,16,65.16,null,null,null", - "65,f,100,25.0,null,65,null", - "100,e,65,16.25,null,100,null", - "200,h,100,16.65,100,null,null" - }; - checkResult(receiverStatement, "select ** from root", columnNames, results, true); - } - - private void checkResultWithDeletion() throws Exception { - String[] columnNames = - new String[] { - "root.sg1.d1.s3", "root.sg1.d1.s1", "root.sg1.d1.s2", - }; - String[] results = - new String[] { - "3,null,null,65.25", - "16,null,25,100.0", - "25,null,16,65.16", - "65,null,100,25.0", - "100,e,65,16.25", - "200,h,100,16.65" - }; - checkResult(receiverStatement, "select ** from root", columnNames, results, true); - } - - @Test - public void testCreatePipe() throws Exception { - preparePipe(); - checkResult( - receiverStatement, - "show pipe", - new String[] {"name", "role", "status"}, - new String[] {"p,receiver,STOP"}, - false); - dropPipe(); - checkResult( - senderStatement, - "show pipe", - new String[] {"name", "role", "remote", "status"}, - new String[] {"p,sender,my_iotdb,DROP"}, - false); - } - - @Test - public void testHistoryInsert() { - try { - prepareSchema(); - prepareIns1(); - prepareIns2(); - prepareIns3(); - preparePipe(); - startPipe(); - checkResult(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testHistoryAndRealTimeInsert() { - try { - prepareSchema(); - prepareIns1(); - prepareIns2(); - preparePipe(); - startPipe(); - prepareIns3(); - checkResult(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testStopAndStartInsert() { - try { - prepareSchema(); - prepareIns1(); - preparePipe(); - startPipe(); - prepareIns2(); - stopPipe(); - prepareIns3(); - startPipe(); - checkResult(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testRealTimeAndStopInsert() { - try { - preparePipe(); // realtime - startPipe(); - prepareSchema(); - prepareIns1(); - stopPipe(); - prepareIns2(); - startPipe(); - prepareIns3(); - checkResult(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testHistoryDel() { - try { - prepareSchema(); // history - prepareIns1(); - prepareIns2(); - prepareIns3(); - prepareDel1(); - prepareDel2(); - prepareDel3(); - preparePipe(); // realtime - startPipe(); - checkResultWithDeletion(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - } - - @Test - public void testRealtimeDel() { - try { - prepareSchema(); // history - prepareIns1(); - preparePipe(); // realtime - startPipe(); - prepareIns2(); - prepareDel1(); - stopPipe(); - prepareIns3(); - startPipe(); - prepareDel2(); - prepareDel3(); - checkResultWithDeletion(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); - } - } - - /** - * Execute sql in IoTDB and compare resultSet with expected result. This method only check columns - * that is explicitly declared in columnNames. This method will compare expected result with - * actual result RETRY_TIME times. Interval of each run is 1000ms. - * - * @param statement Statement of IoTDB. - * @param sql SQL to be executed. - * @param columnNames Columns to be compared with. - * @param retArray Expected result set. Order of columns is as same as columnNames. - * @param hasTimeColumn If result set contains time column (e.g. timeserires query), set - * hasTimeColumn = true. - */ - private static void checkResult( - Statement statement, - String sql, - String[] columnNames, - String[] retArray, - boolean hasTimeColumn) - throws Exception { - loop: - for (int loop = 0; loop < RETRY_TIME; loop++) { - try { - Thread.sleep(1000); - boolean hasResultSet = statement.execute(sql); - if (!assertOrCompareEqual(true, hasResultSet, loop)) { - continue; - } - ResultSet resultSet = statement.getResultSet(); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - Map<String, Integer> map = new HashMap<>(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - map.put(resultSetMetaData.getColumnName(i), i); - } - int cnt = 0; - while (resultSet.next()) { - StringBuilder builder = new StringBuilder(); - if (hasTimeColumn) { - builder.append(resultSet.getString(1)).append(","); - } - for (String columnName : columnNames) { - int index = map.get(columnName); - builder.append(resultSet.getString(index)).append(","); - } - if (builder.length() > 0) { - builder.deleteCharAt(builder.length() - 1); - } - if (!assertOrCompareEqual(retArray[cnt], builder.toString(), loop)) { - continue loop; - } - cnt++; - } - if (!assertOrCompareEqual(retArray.length, cnt, loop)) { - continue; - } - return; - } catch (Exception e) { - if (loop == RETRY_TIME - 1) { - throw e; - } - } - } - Assert.fail(); - } - - private static boolean assertOrCompareEqual(Object expected, Object actual, int loop) { - if (loop == RETRY_TIME - 1) { - assertEquals(expected, actual); - return true; - } else { - return expected.equals(actual); - } - } -} diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncWeakNetworkIT.java b/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncWeakNetworkIT.java index 09effada3a..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncWeakNetworkIT.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sync/SyncWeakNetworkIT.java @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.sync; - -import org.junit.Assert; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.ContainerState; - -/** Simulate network delay and loss. */ -public class SyncWeakNetworkIT extends SyncIT { - @Override - public void init() throws Exception { - super.init(); - // set delay is 200±50ms that conform to a normal distribution; - // network packet with 10% loss rate, 10% duplicate rate, 10% reorder rate and 10% corrupt rate; - Container.ExecResult res = - ((ContainerState) environment.getContainerByServiceName("iotdb-sender_1").get()) - .execInContainer( - "sh", - "-c", - "tc qdisc add dev eth0 root netem delay 200ms 50ms 25% distribution normal loss random 10% duplicate 10% reorder 10% corrupt 10%"); - Assert.assertEquals(0, res.getExitCode()); - } -} diff --git a/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java index 0d1fa76a83..e69de29bb2 100644 --- a/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java +++ b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.testcontainers.containers; - -import java.io.File; -import java.lang.reflect.Field; - -public class NoProjectNameDockerComposeContainer extends DockerComposeContainer { - - public NoProjectNameDockerComposeContainer(String identifier, File... composeFiles) { - super(identifier, composeFiles); - Field project = null; - try { - project = DockerComposeContainer.class.getDeclaredField("project"); - project.setAccessible(true); - project.set(this, ""); - } catch (NoSuchFieldException | IllegalAccessException e) { - e.printStackTrace(); - } - } -} diff --git a/testcontainer/src/test/resources/logback-container.xml b/testcontainer/src/test/resources/logback-container.xml index e897fe24cb..e69de29bb2 100644 --- a/testcontainer/src/test/resources/logback-container.xml +++ b/testcontainer/src/test/resources/logback-container.xml @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<configuration debug="false"> - <property name="LOG_PATH" value="target/logs"/> - <!-- prevent logback from outputting its own status at the start of every log --> - <statusListener class="ch.qos.logback.core.status.NopStatusListener"/> - <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout"> - <Target>System.out</Target> - <encoder> - <pattern>%-5p [%d] [%thread] %C{25}:%L - %m %n</pattern> - <charset>utf-8</charset> - </encoder> - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>DEBUG</level> - </filter> - </appender> - <root level="INFO"> - <appender-ref ref="stdout"/> - </root> -</configuration> diff --git a/testcontainer/src/test/resources/logback.xml b/testcontainer/src/test/resources/logback.xml index 098f463a66..e69de29bb2 100644 --- a/testcontainer/src/test/resources/logback.xml +++ b/testcontainer/src/test/resources/logback.xml @@ -1,52 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<configuration debug="false"> - <property name="LOG_PATH" value="target/logs"/> - <!-- prevent logback from outputting its own status at the start of every log --> - <statusListener class="ch.qos.logback.core.status.NopStatusListener"/> - <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout"> - <Target>System.out</Target> - <encoder> - <pattern>%-5p [%d] [%thread] %C{25}:%L - %m %n</pattern> - <charset>utf-8</charset> - </encoder> - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>DEBUG</level> - </filter> - </appender> - <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout"> - <Target>System.out</Target> - <encoder> - <pattern>[%thread] %m %n</pattern> - <charset>utf-8</charset> - </encoder> - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>DEBUG</level> - </filter> - </appender> - <logger name="org.testcontainers.containers.output.Slf4jLogConsumer" level="INFO"> - <appender-ref ref="Container"/> - </logger> - <root level="INFO"> - <appender-ref ref="stdout"/> - </root> -</configuration> diff --git a/testcontainer/src/test/resources/sync/docker-compose.yaml b/testcontainer/src/test/resources/sync/docker-compose.yaml index fa0cedc8a4..e69de29bb2 100644 --- a/testcontainer/src/test/resources/sync/docker-compose.yaml +++ b/testcontainer/src/test/resources/sync/docker-compose.yaml @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -version: '3.8' - -services: - iotdb-sender: - image: apache/iotdb:sync-maven-development - networks: - - iotdb - cap_add: - - NET_ADMIN - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 - volumes: - - ../logback-container.xml:/iotdb/conf/logback.xml - iotdb-receiver: - image: apache/iotdb:sync-maven-development - networks: - - iotdb - cap_add: - - NET_ADMIN - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 - volumes: - - ../logback-container.xml:/iotdb/conf/logback.xml - - -networks: - iotdb: - driver: bridge \ No newline at end of file diff --git a/testcontainer/src/tool/README.md b/testcontainer/src/tool/README.md index 3b943af7ff..e69de29bb2 100644 --- a/testcontainer/src/tool/README.md +++ b/testcontainer/src/tool/README.md @@ -1,33 +0,0 @@ -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> - -## Motivation -The current E2E framework writes all process logs into a single test log. This approach makes it difficult to find the root cause when there are failed tests with unstable recurrence. So we need a tool that can find the failed tests in this log and separate the logs from the different nodes. - -## Usage -1. Download log archive from CI. - -2. Parse log. -``` -python3 parser.py [filename] -``` - -3. View the separated logs in the current directory. \ No newline at end of file diff --git a/testcontainer/src/tool/parser.py b/testcontainer/src/tool/parser.py index 5cee0a85a0..e69de29bb2 100644 --- a/testcontainer/src/tool/parser.py +++ b/testcontainer/src/tool/parser.py @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import os -import sys -import re - -pattern = re.compile(r"docker\-java\-stream\-+(\d+)") - - -def getAllLogs(filename): - with open(filename, "r") as f: - data = f.read() - return data.split("up -d")[1:] - - -def writeAllLogs(filename, content): - with open(filename, "w") as f: - for row in content: - f.write(row) - f.write("\n") - - -def getNodes(log): - ids = pattern.findall(log) - nodes = {} - for id in ids: - if not nodes.__contains__(id): - nodes[id] = [] - return nodes - - -def checkAndMkdir(i): - if not os.path.exists("./{}".format(i)): - os.mkdir("./{}".format(i)) - - -def parse(nodes, rows): - for row in rows: - for id, content in nodes.items(): - if row.__contains__(id): - content.append(row) - if row.__contains__("[ERROR]"): - for content in nodes.values(): - content.append(row) - - -def output(nodes, i): - for key, content in nodes.items(): - writeAllLogs("./{}/{}_{}.txt".format(i, i, key), content) - - -if __name__ == "__main__": - logs = getAllLogs(sys.argv[1]) - count = 0 - for i in range(len(logs)): - if logs[i].__contains__("FAILURE!"): - nodes = getNodes(logs[i]) - parse(nodes, logs[i].split("\n")) - checkAndMkdir(i) - output(nodes, i) - count = count + 1 - - print("find {} failed tests".format(count))
