This is an automated email from the ASF dual-hosted git repository.
luoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 9a9005a DRILL-7911 Use TestContainers-MySQL instead of
wix-embedded-mysql
9a9005a is described below
commit 9a9005aa5eb41e07c11a935521f8635f7b79b6c5
Author: Martin Tzvetanov Grigorov <[email protected]>
AuthorDate: Wed May 5 15:59:45 2021 +0300
DRILL-7911 Use TestContainers-MySQL instead of wix-embedded-mysql
Wix Embedded MySQL is not actively maintained since Oct 2019. And it uses
old versions of Flapdoodle.process so it does not support ARM64.
Since there is no ARM64 Docker image for mysql we use mariadb as a
substitute on Linux ARM64
---
contrib/storage-cassandra/pom.xml | 2 +-
contrib/storage-jdbc/pom.xml | 9 ++--
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 62 ++++++++++++----------
.../mysql_config_override/mysql_override.cnf | 2 +
contrib/storage-mongo/pom.xml | 2 +-
contrib/storage-splunk/pom.xml | 2 +-
exec/java-exec/pom.xml | 2 +-
pom.xml | 15 +++++-
8 files changed, 58 insertions(+), 38 deletions(-)
diff --git a/contrib/storage-cassandra/pom.xml
b/contrib/storage-cassandra/pom.xml
index facfb5e..ee8cf6e 100644
--- a/contrib/storage-cassandra/pom.xml
+++ b/contrib/storage-cassandra/pom.xml
@@ -82,7 +82,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>cassandra</artifactId>
- <version>1.15.1</version>
+ <version>${testcontainers.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
index c709337..916074d 100755
--- a/contrib/storage-jdbc/pom.xml
+++ b/contrib/storage-jdbc/pom.xml
@@ -31,8 +31,7 @@
<name>Drill : Contrib : Storage : JDBC</name>
<properties>
- <mysql.connector.version>8.0.19</mysql.connector.version>
- <wix-embedded-mysql.version>4.6.1</wix-embedded-mysql.version>
+ <mysql.connector.version>8.0.25</mysql.connector.version>
<h2.version>1.4.200</h2.version>
</properties>
@@ -69,9 +68,9 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.wix</groupId>
- <artifactId>wix-embedded-mysql</artifactId>
- <version>${wix-embedded-mysql.version}</version>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ <version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git
a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
index fab2faa..2775439 100644
---
a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
+++
b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
@@ -17,11 +17,6 @@
*/
package org.apache.drill.exec.store.jdbc;
-import com.wix.mysql.EmbeddedMysql;
-import com.wix.mysql.ScriptResolver;
-import com.wix.mysql.config.MysqldConfig;
-import com.wix.mysql.config.SchemaConfig;
-import com.wix.mysql.distribution.Version;
import org.apache.drill.categories.JdbcStorageTest;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
@@ -31,14 +26,17 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
-import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.test.rowSet.RowSetUtilities;
-import org.joda.time.DateTimeZone;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.testcontainers.containers.JdbcDatabaseContainer;
+import org.testcontainers.containers.MySQLContainer;
+import org.testcontainers.ext.ScriptUtils;
+import org.testcontainers.jdbc.JdbcDatabaseDelegate;
+import org.testcontainers.utility.DockerImageName;
import java.math.BigDecimal;
@@ -51,42 +49,50 @@ import static org.junit.Assert.assertEquals;
@Category(JdbcStorageTest.class)
public class TestJdbcPluginWithMySQLIT extends ClusterTest {
- private static EmbeddedMysql mysqld;
+ private static final String DOCKER_IMAGE_MYSQL = "mysql:5.7.27";
+ private static final String DOCKER_IMAGE_MARIADB = "mariadb:10.6.0";
+ private static JdbcDatabaseContainer<?> jdbcContainer;
@BeforeClass
public static void initMysql() throws Exception {
startCluster(ClusterFixture.builder(dirTestWatcher));
+ String osName = System.getProperty("os.name").toLowerCase();
String mysqlDBName = "drill_mysql_test";
- int mysqlPort = QueryTestUtil.getFreePortNumber(2215, 300);
- MysqldConfig config = MysqldConfig.aMysqldConfig(Version.v5_7_27)
- .withPort(mysqlPort)
- .withUser("mysqlUser", "mysqlPass")
- .withTimeZone(DateTimeZone.UTC.toTimeZone())
- .build();
+ DockerImageName imageName;
+ if (osName.startsWith("linux") &&
"aarch64".equals(System.getProperty("os.arch"))) {
+ imageName =
DockerImageName.parse(DOCKER_IMAGE_MARIADB).asCompatibleSubstituteFor("mysql");
+ } else {
+ imageName = DockerImageName.parse(DOCKER_IMAGE_MYSQL);
+ }
- SchemaConfig.Builder schemaConfig = SchemaConfig.aSchemaConfig(mysqlDBName)
- .withScripts(ScriptResolver.classPathScript("mysql-test-data.sql"));
+ jdbcContainer = new MySQLContainer<>(imageName)
+ .withExposedPorts(3306)
+ .withConfigurationOverride("mysql_config_override")
+ .withUsername("mysqlUser")
+ .withPassword("mysqlPass")
+ .withDatabaseName(mysqlDBName)
+ .withUrlParam("serverTimezone", "UTC")
+ .withUrlParam("useJDBCCompliantTimezoneShift", "true")
+ .withInitScript("mysql-test-data.sql");
+ jdbcContainer.start();
- String osName = System.getProperty("os.name").toLowerCase();
if (osName.startsWith("linux")) {
-
schemaConfig.withScripts(ScriptResolver.classPathScript("mysql-test-data-linux.sql"));
+ JdbcDatabaseDelegate databaseDelegate = new
JdbcDatabaseDelegate(jdbcContainer, "");
+ ScriptUtils.runInitScript(databaseDelegate, "mysql-test-data-linux.sql");
}
- mysqld =
EmbeddedMysql.anEmbeddedMysql(config).addSchema(schemaConfig.build()).start();
-
- JdbcStorageConfig jdbcStorageConfig = new
JdbcStorageConfig("com.mysql.cj.jdbc.Driver",
-
String.format("jdbc:mysql://localhost:%s/%s?useJDBCCompliantTimezoneShift=true",
mysqlPort, mysqlDBName),
- "mysqlUser", "mysqlPass", false, null, null);
+ String jdbcUrl = jdbcContainer.getJdbcUrl();
+ JdbcStorageConfig jdbcStorageConfig = new
JdbcStorageConfig("com.mysql.cj.jdbc.Driver", jdbcUrl,
+ jdbcContainer.getUsername(), jdbcContainer.getPassword(), false,
null, null);
jdbcStorageConfig.setEnabled(true);
cluster.defineStoragePlugin("mysql", jdbcStorageConfig);
if (osName.startsWith("linux")) {
// adds storage plugin with case insensitive table names
- JdbcStorageConfig jdbcCaseSensitiveStorageConfig = new
JdbcStorageConfig("com.mysql.cj.jdbc.Driver",
-
String.format("jdbc:mysql://localhost:%s/%s?useJDBCCompliantTimezoneShift=true",
mysqlPort, mysqlDBName),
- "mysqlUser", "mysqlPass", true, null, null);
+ JdbcStorageConfig jdbcCaseSensitiveStorageConfig = new
JdbcStorageConfig("com.mysql.cj.jdbc.Driver", jdbcUrl,
+ jdbcContainer.getUsername(), jdbcContainer.getPassword(), true,
null, null);
jdbcCaseSensitiveStorageConfig.setEnabled(true);
cluster.defineStoragePlugin("mysqlCaseInsensitive",
jdbcCaseSensitiveStorageConfig);
}
@@ -94,8 +100,8 @@ public class TestJdbcPluginWithMySQLIT extends ClusterTest {
@AfterClass
public static void stopMysql() {
- if (mysqld != null) {
- mysqld.stop();
+ if (jdbcContainer != null) {
+ jdbcContainer.stop();
}
}
diff --git
a/contrib/storage-jdbc/src/test/resources/mysql_config_override/mysql_override.cnf
b/contrib/storage-jdbc/src/test/resources/mysql_config_override/mysql_override.cnf
new file mode 100644
index 0000000..dd15b98
--- /dev/null
+++
b/contrib/storage-jdbc/src/test/resources/mysql_config_override/mysql_override.cnf
@@ -0,0 +1,2 @@
+[mysqld]
+tls_version=TLSv1.2
diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml
index fdd6204..90f467f 100644
--- a/contrib/storage-mongo/pom.xml
+++ b/contrib/storage-mongo/pom.xml
@@ -66,7 +66,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
- <version>1.15.2</version>
+ <version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/contrib/storage-splunk/pom.xml b/contrib/storage-splunk/pom.xml
index 665a3c9..9ce6cca 100644
--- a/contrib/storage-splunk/pom.xml
+++ b/contrib/storage-splunk/pom.xml
@@ -65,7 +65,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
- <version>1.15.1</version>
+ <version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 65d7db4..27180ef 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -621,7 +621,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>vault</artifactId>
- <version>1.15.1</version>
+ <version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/pom.xml b/pom.xml
index f0ae03b..8959bc1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,6 +111,7 @@
<joda.version>2.10.5</joda.version>
<javax.el.version>3.0.0</javax.el.version>
<surefire.version>3.0.0-M5</surefire.version>
+ <jna.version>5.8.0</jna.version>
<commons.compress.version>1.20</commons.compress.version>
<hikari.version>3.4.2</hikari.version>
<netty.version>4.1.59.Final</netty.version>
@@ -120,6 +121,7 @@
<commons.cli.version>1.4</commons.cli.version>
<snakeyaml.version>1.26</snakeyaml.version>
<commons.lang3.version>3.10</commons.lang3.version>
+ <testcontainers.version>1.15.3</testcontainers.version>
<typesafe.config.version>1.0.0</typesafe.config.version>
<commons.codec.version>1.14</commons.codec.version>
<metadata.extractor.version>2.13.0</metadata.extractor.version>
@@ -450,6 +452,7 @@
<exclude>**/*.prj</exclude>
<exclude>**/*.shp</exclude>
<exclude>**/*.dbf</exclude>
+ <exclude>**/*.cnf</exclude>
</excludes>
</configuration>
</plugin>
@@ -1164,6 +1167,16 @@
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>${jna.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>${jna.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
@@ -1728,7 +1741,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative</artifactId>
- <version>2.0.1.Final</version>
+ <version>2.0.39.Final</version>
<classifier>${netty.tcnative.classifier}</classifier>
<scope>runtime</scope>
<optional>true</optional>
