This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 9a9917d37c [Improve][E2E] improve doris initialize (#8446)
9a9917d37c is described below
commit 9a9917d37c961f9e5523fb754d9a5831135a3473
Author: zhangdonghao <[email protected]>
AuthorDate: Sat Jan 4 15:52:09 2025 +0800
[Improve][E2E] improve doris initialize (#8446)
---
.../e2e/connector/doris/AbstractDorisIT.java | 40 ++++++++++++++--------
.../e2e/connector/doris/DorisErrorIT.java | 2 +-
.../seatunnel/e2e/connector/doris/DorisIT.java | 11 ++++++
.../e2e/connector/doris/DorisNoSchemaIT.java | 38 --------------------
.../e2e/connector/doris/GenerateTestData.java | 6 ----
5 files changed, 38 insertions(+), 59 deletions(-)
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java
index 19e17d8089..d8647f0f07 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java
@@ -25,9 +25,7 @@ import org.apache.seatunnel.e2e.common.TestSuiteBase;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
-import org.testcontainers.utility.DockerLoggerFactory;
import lombok.extern.slf4j.Slf4j;
@@ -40,6 +38,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
@@ -71,17 +71,18 @@ public abstract class AbstractDorisIT extends TestSuiteBase
implements TestResou
protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
protected static final String DRIVER_JAR =
"https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar";;
+ private static final boolean isGithubActionsEnv =
+ "true".equalsIgnoreCase(System.getenv("GITHUB_ACTIONS"));
@BeforeAll
@Override
public void startUp() {
+ log.info("isGithubActionsEnv: {}", isGithubActionsEnv);
container =
new GenericContainer<>(DOCKER_IMAGE)
.withNetwork(NETWORK)
.withNetworkAliases(HOST)
- .withPrivilegedMode(true)
- .withLogConsumer(
- new
Slf4jLogConsumer(DockerLoggerFactory.getLogger(DOCKER_IMAGE)));
+ .withPrivilegedMode(true);
container.setPortBindings(
Lists.newArrayList(
String.format("%s:%s", QUERY_PORT, QUERY_PORT),
@@ -89,15 +90,17 @@ public abstract class AbstractDorisIT extends TestSuiteBase
implements TestResou
String.format("%s:%s", BE_HTTP_PORT, BE_HTTP_PORT)));
Startables.deepStart(Stream.of(container)).join();
log.info("doris container started");
- given().ignoreExceptions()
+ given().pollDelay(20, TimeUnit.SECONDS)
.await()
.atMost(360, TimeUnit.SECONDS)
.untilAsserted(this::initializeJdbcConnection);
+ log.info("doris initialized");
}
protected void initializeJdbcConnection()
throws SQLException, ClassNotFoundException, MalformedURLException,
InstantiationException, IllegalAccessException {
+ log.info("doris initializing ...");
URLClassLoader urlClassLoader =
new URLClassLoader(new URL[] {new URL(DRIVER_JAR)},
DorisIT.class.getClassLoader());
Thread.currentThread().setContextClassLoader(urlClassLoader);
@@ -106,7 +109,9 @@ public abstract class AbstractDorisIT extends TestSuiteBase
implements TestResou
props.put("user", USERNAME);
props.put("password", PASSWORD);
jdbcConnection = driver.connect(String.format(URL,
container.getHost()), props);
- initializeBE();
+ if (isGithubActionsEnv) {
+ initializeBE();
+ }
try (Statement statement = jdbcConnection.createStatement()) {
statement.execute(SET_SQL);
statement.execute(SET_CONNECTIONS);
@@ -124,14 +129,21 @@ public abstract class AbstractDorisIT extends
TestSuiteBase implements TestResou
// cross-container access failure. Delete the BE and add it again
private void initializeBE() {
try (Statement statement = jdbcConnection.createStatement()) {
- ResultSet resultSet = statement.executeQuery(SHOW_FE);
- String feIp = null;
- while (resultSet.next()) {
- feIp = resultSet.getString("Host");
+ ResultSet beResultSet = statement.executeQuery(SHOW_BE);
+ List<String> beList = new ArrayList<>();
+ while (beResultSet.next()) {
+ beList.add(beResultSet.getString("Host"));
+ }
+ if (beList.stream().anyMatch("127.0.0.1"::equals)) {
+ ResultSet resultSet = statement.executeQuery(SHOW_FE);
+ String feIp = null;
+ while (resultSet.next()) {
+ feIp = resultSet.getString("Host");
+ }
+ statement.execute(DROP_BE);
+ statement.execute(String.format(ADD_BE, feIp));
+ log.info("doris BE initialized");
}
- statement.execute(DROP_BE);
- statement.execute(String.format(ADD_BE, feIp));
-
} catch (SQLException e) {
throw new RuntimeException(e);
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java
index d255c917c3..517b600533 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java
@@ -90,7 +90,7 @@ public class DorisErrorIT extends AbstractDorisIT {
log.info("doris error log: \n" + future.get().getStderr());
super.container.start();
// wait for the container to restart
- given().ignoreExceptions()
+ given().pollInterval(20, TimeUnit.SECONDS)
.await()
.atMost(360, TimeUnit.SECONDS)
.untilAsserted(this::initializeJdbcConnection);
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java
index 178ed0ffba..d817c7743b 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java
@@ -202,6 +202,16 @@ public class DorisIT extends AbstractDorisIT {
checkAllTypeSinkData();
}
+ @TestTemplate
+ public void testNoSchemaDoris(TestContainer container)
+ throws IOException, InterruptedException {
+ initializeJdbcTable();
+ batchInsertUniqueTableData();
+ Container.ExecResult execResult1 =
container.executeJob("/doris_source_no_schema.conf");
+ Assertions.assertEquals(0, execResult1.getExitCode());
+ checkSinkData();
+ }
+
private void checkAllTypeSinkData() {
try {
assertHasData(sourceDB, DUPLICATE_TABLE);
@@ -412,6 +422,7 @@ public class DorisIT extends AbstractDorisIT {
// create source and sink table
statement.execute(createUniqueTableForTest(sourceDB));
statement.execute(createDuplicateTableForTest(sourceDB));
+ log.info("create source and sink table succeed");
} catch (SQLException e) {
throw new RuntimeException("Initializing table failed!", e);
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisNoSchemaIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisNoSchemaIT.java
deleted file mode 100644
index a2806ae9ab..0000000000
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisNoSchemaIT.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.e2e.connector.doris;
-
-import org.apache.seatunnel.e2e.common.container.TestContainer;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.TestTemplate;
-import org.testcontainers.containers.Container;
-
-import java.io.IOException;
-
-public class DorisNoSchemaIT extends DorisIT {
-
- @TestTemplate
- public void testDoris(TestContainer container) throws IOException,
InterruptedException {
- initializeJdbcTable();
- batchInsertUniqueTableData();
- Container.ExecResult execResult1 =
container.executeJob("/doris_source_no_schema.conf");
- Assertions.assertEquals(0, execResult1.getExitCode());
- checkSinkData();
- }
-}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/GenerateTestData.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/GenerateTestData.java
index 40ce1a9ad6..f2defc8716 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/GenerateTestData.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/GenerateTestData.java
@@ -129,10 +129,4 @@ public class GenerateTestData {
return integer.add(decimal).setScale(decimalDigits,
RoundingMode.HALF_UP);
}
-
- public static void main(String[] args) {
- for (int i = 0; i < 1000; i++) {
- System.out.println(genFloat(0, 1000));
- }
- }
}
