This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new f0e0395b36 [Chore] Use testcontainer origin exposed port rather than
generate random port (#16195)
f0e0395b36 is described below
commit f0e0395b36bba5977135a3b7b4472b91ec388e50
Author: Wenjun Ruan <[email protected]>
AuthorDate: Fri Jun 21 22:19:54 2024 +0800
[Chore] Use testcontainer origin exposed port rather than generate random
port (#16195)
---
.../dao/repository/impl/CommandDaoImplTest.java | 17 +++++------------
.../plugin/registry/etcd/EtcdRegistryTestCase.java | 1 +
.../registry/jdbc/MysqlJdbcRegistryTestCase.java | 10 +++-------
.../jdbc/PostgresqlJdbcRegistryTestCase.java | 9 ++-------
.../zookeeper/ZookeeperRegistryTestCase.java | 12 ++++--------
.../storage/hdfs/LocalStorageOperatorTest.java | 1 +
.../DolphinSchedulerDatabaseContainerExtension.java | 8 --------
.../mysql/DolphinSchedulerMysqlProfile.java | 2 ++
.../mysql/MysqlDatabaseContainerProvider.java | 21 +++++++++++++++++++--
.../DolphinSchedulerPostgresqlProfile.java | 2 ++
.../PostgresqlDatabaseContainerProvider.java | 16 ++++++++++++++--
11 files changed, 53 insertions(+), 46 deletions(-)
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
index 1897ad7fa2..ec12bbb2f2 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
@@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.Command;
-import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.repository.CommandDao;
import org.apache.commons.lang3.RandomUtils;
@@ -39,34 +38,28 @@ import java.util.stream.Collectors;
import org.junit.jupiter.api.RepeatedTest;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.annotation.DirtiesContext;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class CommandDaoImplTest extends BaseDaoTest {
@Autowired
private CommandDao commandDao;
- @Autowired
- private CommandMapper commandMapper;
-
- @RepeatedTest(value = 100)
+ @RepeatedTest(value = 10)
void fetchCommandByIdSlot() {
- // clear all commands
- commandMapper.delete(new QueryWrapper<Command>().ge("id", -1));
-
int totalSlot = RandomUtils.nextInt(1, 10);
int currentSlotIndex = RandomUtils.nextInt(0, totalSlot);
int fetchSize = RandomUtils.nextInt(10, 100);
int idStep = RandomUtils.nextInt(1, 5);
int commandSize = RandomUtils.nextInt(currentSlotIndex, 1000);
// Generate commandSize commands
- int id = 0;
+ int id = 1;
for (int j = 0; j < commandSize; j++) {
- id += idStep;
Command command = generateCommand(CommandType.START_PROCESS, 0);
command.setId(id);
commandDao.insert(command);
+ id += idStep;
}
List<Command> commands =
commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize);
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java
index 1e751c1862..39bfea8cc7 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java
@@ -51,6 +51,7 @@ public class EtcdRegistryTestCase extends
RegistryTestCase<EtcdRegistry> {
.build()
.cluster();
etcdCluster.start();
+ System.clearProperty("registry.endpoints");
System.setProperty("registry.endpoints",
etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")));
}
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java
index 0db601693b..6dca1aeb2a 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.registry.jdbc;
-import org.apache.commons.lang3.RandomUtils;
-
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
@@ -37,8 +35,6 @@ import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
-import com.google.common.collect.Lists;
-
@ActiveProfiles("mysql")
class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
@@ -55,11 +51,11 @@ class MysqlJdbcRegistryTestCase extends
JdbcRegistryTestCase {
.withExposedPorts(3306)
.waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
- int exposedPort = RandomUtils.nextInt(10000, 65535);
- mysqlContainer.setPortBindings(Lists.newArrayList(exposedPort +
":3306"));
Startables.deepStart(Stream.of(mysqlContainer)).join();
- String jdbcUrl = "jdbc:mysql://localhost:" + exposedPort +
"/dolphinscheduler?useSSL=false&serverTimezone=UTC";
+ String jdbcUrl = "jdbc:mysql://localhost:" +
mysqlContainer.getMappedPort(3306)
+ + "/dolphinscheduler?useSSL=false&serverTimezone=UTC";
+ System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
try (
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java
index a86533dcaf..f34015e5a2 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.registry.jdbc;
-import org.apache.commons.lang3.RandomUtils;
-
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
@@ -37,8 +35,6 @@ import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
-import com.google.common.collect.Lists;
-
@ActiveProfiles("postgresql")
@SpringBootTest(classes = {JdbcRegistryProperties.class})
@SpringBootApplication(scanBasePackageClasses = JdbcRegistryProperties.class)
@@ -55,12 +51,11 @@ public class PostgresqlJdbcRegistryTestCase extends
JdbcRegistryTestCase {
.withDatabaseName("dolphinscheduler")
.withNetwork(Network.newNetwork())
.withExposedPorts(5432);
- int exposedPort = RandomUtils.nextInt(10000, 65535);
- postgresqlContainer.setPortBindings(Lists.newArrayList(exposedPort +
":5432"));
Startables.deepStart(Stream.of(postgresqlContainer)).join();
- String jdbcUrl = "jdbc:postgresql://localhost:" + exposedPort +
"/dolphinscheduler";
+ String jdbcUrl = "jdbc:postgresql://localhost:" +
postgresqlContainer.getMappedPort(5432) + "/dolphinscheduler";
+ System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
try (
Connection connection = DriverManager.getConnection(jdbcUrl,
"root", "root");
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java
index 60d8520b81..73c784b2df 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java
@@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper;
import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase;
-import org.apache.commons.lang3.RandomUtils;
-
import java.util.stream.Stream;
import lombok.SneakyThrows;
@@ -35,8 +33,6 @@ import org.testcontainers.containers.Network;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
-import com.google.common.collect.Lists;
-
@SpringBootTest(classes = ZookeeperRegistryProperties.class)
@SpringBootApplication(scanBasePackageClasses =
ZookeeperRegistryProperties.class)
class ZookeeperRegistryTestCase extends RegistryTestCase<ZookeeperRegistry> {
@@ -52,11 +48,11 @@ class ZookeeperRegistryTestCase extends
RegistryTestCase<ZookeeperRegistry> {
@BeforeAll
public static void setUpTestingServer() {
zookeeperContainer = new
GenericContainer<>(DockerImageName.parse("zookeeper:3.8"))
- .withNetwork(NETWORK);
- int randomPort = RandomUtils.nextInt(10000, 65535);
- zookeeperContainer.setPortBindings(Lists.newArrayList(randomPort +
":2181"));
+ .withNetwork(NETWORK)
+ .withExposedPorts(2181);
Startables.deepStart(Stream.of(zookeeperContainer)).join();
- System.setProperty("registry.zookeeper.connect-string", "localhost:" +
randomPort);
+ System.clearProperty("registry.zookeeper.connect-string");
+ System.setProperty("registry.zookeeper.connect-string", "localhost:" +
zookeeperContainer.getMappedPort(2181));
}
@SneakyThrows
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
index 7a277afbce..f6ed76e265 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
@@ -52,6 +52,7 @@ class LocalStorageOperatorTest {
@BeforeEach
public void setup() {
Files.createDirectories(Paths.get(resourceBaseDir));
+ System.clearProperty(Constants.RESOURCE_UPLOAD_PATH);
System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir);
LocalStorageOperatorFactory localStorageOperatorFactory = new
LocalStorageOperatorFactory();
diff --git
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java
index 4b463e1ef9..a264ca09e9 100644
---
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java
+++
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java
@@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.tools.datasource.jupiter;
import java.util.HashMap;
import java.util.Map;
import java.util.ServiceLoader;
-import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
@@ -28,7 +27,6 @@ import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
@Slf4j
@@ -39,12 +37,6 @@ public class DolphinSchedulerDatabaseContainerExtension
implements BeforeAllCall
@Override
public void beforeAll(ExtensionContext context) {
databaseContainer = getDataSourceContainer(context);
- log.info("Create {} successfully.",
databaseContainer.getDockerImageName());
- databaseContainer.start();
-
- log.info("Starting {}...", databaseContainer.getDockerImageName());
- Startables.deepStart(Stream.of(databaseContainer)).join();
- log.info("{} started", databaseContainer.getDockerImageName());
}
diff --git
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java
index f32f4d80d4..a8670d5934 100644
---
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java
+++
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java
@@ -27,6 +27,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
@Inherited
@@ -34,6 +35,7 @@ import org.springframework.test.context.ActiveProfiles;
@Retention(RetentionPolicy.RUNTIME)
@ActiveProfiles("mysql")
@SpringBootTest(classes = {UpgradeDolphinScheduler.class,
DaoConfiguration.class})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
public @interface DolphinSchedulerMysqlProfile {
}
diff --git
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java
index 80d0b8978d..d27a897920 100644
---
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java
+++
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java
@@ -20,15 +20,20 @@ package org.apache.dolphinscheduler.tools.datasource.mysql;
import
org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
import
org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
+import java.util.stream.Stream;
+
+import lombok.extern.slf4j.Slf4j;
+
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.auto.service.AutoService;
-import com.google.common.collect.Lists;
+@Slf4j
@AutoService(DatabaseContainerProvider.class)
public class MysqlDatabaseContainerProvider implements
DatabaseContainerProvider {
@@ -43,7 +48,19 @@ public class MysqlDatabaseContainerProvider implements
DatabaseContainerProvider
.withNetwork(NETWORK)
.withExposedPorts(3306)
.waitingFor(Wait.forHealthcheck());
- mysqlContainer.setPortBindings(Lists.newArrayList("3306:3306"));
+
+ log.info("Create {} successfully.",
mysqlContainer.getDockerImageName());
+ mysqlContainer.start();
+
+ log.info("Starting {}...", mysqlContainer.getDockerImageName());
+ Startables.deepStart(Stream.of(mysqlContainer)).join();
+ log.info("{} started", mysqlContainer.getDockerImageName());
+
+ String jdbcUrl = "jdbc:mysql://localhost:" +
mysqlContainer.getMappedPort(3306)
+ + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
+ System.clearProperty("spring.datasource.url");
+ System.setProperty("spring.datasource.url", jdbcUrl);
+
return mysqlContainer;
}
diff --git
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java
index 1a5a07ad93..73b02f8a76 100644
---
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java
+++
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java
@@ -27,12 +27,14 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
@Inherited
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@ActiveProfiles("postgresql")
@SpringBootTest(classes = {UpgradeDolphinScheduler.class,
DaoConfiguration.class})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
public @interface DolphinSchedulerPostgresqlProfile {
}
diff --git
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java
index c09b7fae01..d8157d7029 100644
---
a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java
+++
b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java
@@ -20,15 +20,17 @@ package
org.apache.dolphinscheduler.tools.datasource.postgresql;
import
org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
import
org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
+import java.util.stream.Stream;
+
import lombok.extern.slf4j.Slf4j;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.auto.service.AutoService;
-import com.google.common.collect.Lists;
@Slf4j
@AutoService(DatabaseContainerProvider.class)
@@ -47,8 +49,18 @@ public class PostgresqlDatabaseContainerProvider implements
DatabaseContainerPro
.withDatabaseName("dolphinscheduler")
.withNetwork(NETWORK)
.withExposedPorts(5432);
- postgresqlContainer.setPortBindings(Lists.newArrayList("5432:5432"));
+ log.info("Create {} successfully.",
postgresqlContainer.getDockerImageName());
+ postgresqlContainer.start();
+
+ log.info("Starting {}...", postgresqlContainer.getDockerImageName());
+ Startables.deepStart(Stream.of(postgresqlContainer)).join();
+ log.info("{} started", postgresqlContainer.getDockerImageName());
+
+ String jdbcUrl = "jdbc:mysql://localhost:" +
postgresqlContainer.getMappedPort(5432)
+ + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
+ System.clearProperty("spring.datasource.url");
+ System.setProperty("spring.datasource.url", jdbcUrl);
return postgresqlContainer;
}
