This is an automated email from the ASF dual-hosted git repository. davidlim pushed a commit to branch 0.13.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
commit 51ff7f6742e73ed6e919a6f8345b24f5ab66b3b7 Author: David Lim <[email protected]> AuthorDate: Sat Oct 20 19:18:58 2018 -0600 include mysql-metadata-storage extension in distribution, but without… (#6497) * include mysql-metadata-storage extension in distribution, but without the GPL-licensed connector library * Install mysql connector package * use symlinks to avoid versioning issues * add documentation for fetching the mysql connector --- distribution/pom.xml | 19 ------------------- distribution/src/assembly/assembly.xml | 3 --- docs/content/development/extensions-core/mysql.md | 18 ++++++++++++++++++ extensions-core/mysql-metadata-storage/pom.xml | 1 + .../druid/metadata/storage/mysql/MySQLConnector.java | 15 ++++++++++++++- integration-tests/docker/Dockerfile | 5 +++++ integration-tests/docker/broker.conf | 2 +- integration-tests/docker/coordinator.conf | 2 +- integration-tests/docker/historical.conf | 2 +- integration-tests/docker/middlemanager.conf | 2 +- integration-tests/docker/overlord.conf | 2 +- .../docker/router-no-client-auth-tls.conf | 2 +- integration-tests/docker/router-permissive-tls.conf | 2 +- integration-tests/docker/router.conf | 2 +- 14 files changed, 46 insertions(+), 31 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index d9d7bc7..c7706cc 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -164,25 +164,6 @@ </arguments> </configuration> </execution> - <execution> - <id>mysql-tarball</id> - <phase>package</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>tar</executable> - <arguments> - <argument>-C</argument> - <argument>${project.build.directory}/extensions</argument> - <argument>-czvf</argument> - <argument> - ${project.build.directory}/mysql-metadata-storage-${project.parent.version}.tar.gz - </argument> - <argument>mysql-metadata-storage</argument> - </arguments> - </configuration> - </execution> </executions> </plugin> <plugin> diff --git a/distribution/src/assembly/assembly.xml b/distribution/src/assembly/assembly.xml index c1251f4..ab3e5e0 100644 --- a/distribution/src/assembly/assembly.xml +++ b/distribution/src/assembly/assembly.xml @@ -31,9 +31,6 @@ <includes> <include>*/*</include> </includes> - <excludes> - <exclude>mysql-metadata-storage/**</exclude> - </excludes> <outputDirectory>extensions</outputDirectory> </fileSet> diff --git a/docs/content/development/extensions-core/mysql.md b/docs/content/development/extensions-core/mysql.md index a51b84f..09bec90 100644 --- a/docs/content/development/extensions-core/mysql.md +++ b/docs/content/development/extensions-core/mysql.md @@ -6,6 +6,24 @@ layout: doc_page Make sure to [include](../../operations/including-extensions.html) `mysql-metadata-storage` as an extension. +<div class="note caution"> +The MySQL extension requires the MySQL Connector/J library which is not included in the Druid distribution. +Refer to the following section for instructions on how to install this library. +</div> + +## Installing the MySQL connector library + +This extension uses Oracle's MySQL JDBC driver which is not included in the Druid distribution and must be +installed separately. There are a few ways to obtain this library: + +- It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/ +- It can be fetched from Maven Central at: http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar +- It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS + +This should fetch a JAR file named similar to 'mysql-connector-java-x.x.xx.jar'. + +Copy or symlink this file to `extensions/mysql-metadata-storage` under the distribution root directory. + ## Setting up MySQL 1. Install MySQL diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 60bee90..a59aaf7 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -57,6 +57,7 @@ <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.38</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.jdbi</groupId> diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java index 8778eb3..ffb9567 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java @@ -45,6 +45,7 @@ public class MySQLConnector extends SQLMetadataConnector private static final String PAYLOAD_TYPE = "LONGBLOB"; private static final String SERIAL_TYPE = "BIGINT(20) AUTO_INCREMENT"; private static final String QUOTE_STRING = "`"; + private static final String MYSQL_JDBC_DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; private final DBI dbi; @@ -57,11 +58,23 @@ public class MySQLConnector extends SQLMetadataConnector { super(config, dbTables); + try { + Class.forName(MYSQL_JDBC_DRIVER_CLASS_NAME, false, getClass().getClassLoader()); + } + catch (ClassNotFoundException e) { + throw new ISE(e, "Could not find %s on the classpath. The MySQL Connector library is not included in the Druid " + + "distribution but is required to use MySQL. Please download a compatible library (for example " + + "'mysql-connector-java-5.1.38.jar') and place it under 'extensions/mysql-metadata-storage/'. See " + + "https://druid.apache.org/downloads for more details.", + MYSQL_JDBC_DRIVER_CLASS_NAME + ); + } + final BasicDataSource datasource = getDatasource(); // MySQL driver is classloader isolated as part of the extension // so we need to help JDBC find the driver datasource.setDriverClassLoader(getClass().getClassLoader()); - datasource.setDriverClassName("com.mysql.jdbc.Driver"); + datasource.setDriverClassName(MYSQL_JDBC_DRIVER_CLASS_NAME); datasource.addConnectionProperty("useSSL", String.valueOf(connectorConfig.isUseSSL())); if (connectorConfig.isUseSSL()) { log.info("SSL is enabled on this MySQL connection. "); diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 5cbb54c..35deb9a 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -25,6 +25,11 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \ # Add Druid jars ADD lib/* /usr/local/druid/lib/ +# Download the MySQL Java connector +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java +RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar + # Add sample data # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72. RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \ diff --git a/integration-tests/docker/broker.conf b/integration-tests/docker/broker.conf index 4f49dfa..39b8a7f 100644 --- a/integration-tests/docker/broker.conf +++ b/integration-tests/docker/broker.conf @@ -57,7 +57,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server broker redirect_stderr=true autorestart=false diff --git a/integration-tests/docker/coordinator.conf b/integration-tests/docker/coordinator.conf index 4fe113b..e05c924 100644 --- a/integration-tests/docker/coordinator.conf +++ b/integration-tests/docker/coordinator.conf @@ -51,7 +51,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server coordinator redirect_stderr=true priority=100 diff --git a/integration-tests/docker/historical.conf b/integration-tests/docker/historical.conf index 1c4fb96..1b3a2ce 100644 --- a/integration-tests/docker/historical.conf +++ b/integration-tests/docker/historical.conf @@ -53,7 +53,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server historical redirect_stderr=true priority=100 diff --git a/integration-tests/docker/middlemanager.conf b/integration-tests/docker/middlemanager.conf index 50985d0..efe4dec 100644 --- a/integration-tests/docker/middlemanager.conf +++ b/integration-tests/docker/middlemanager.conf @@ -58,7 +58,7 @@ command=java -Ddruid.client.https.keyStorePassword=druid123 -Ddruid.startup.logging.logProperties=true -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server middleManager redirect_stderr=true priority=100 diff --git a/integration-tests/docker/overlord.conf b/integration-tests/docker/overlord.conf index 304eb43..77514b1 100644 --- a/integration-tests/docker/overlord.conf +++ b/integration-tests/docker/overlord.conf @@ -52,7 +52,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server overlord redirect_stderr=true priority=100 diff --git a/integration-tests/docker/router-no-client-auth-tls.conf b/integration-tests/docker/router-no-client-auth-tls.conf index 804a355..fb83b48 100644 --- a/integration-tests/docker/router-no-client-auth-tls.conf +++ b/integration-tests/docker/router-no-client-auth-tls.conf @@ -47,7 +47,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server router redirect_stderr=true priority=100 diff --git a/integration-tests/docker/router-permissive-tls.conf b/integration-tests/docker/router-permissive-tls.conf index 1453c4b..62932c7 100644 --- a/integration-tests/docker/router-permissive-tls.conf +++ b/integration-tests/docker/router-permissive-tls.conf @@ -47,7 +47,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server router redirect_stderr=true priority=100 diff --git a/integration-tests/docker/router.conf b/integration-tests/docker/router.conf index a4a604a..29bb6b5 100644 --- a/integration-tests/docker/router.conf +++ b/integration-tests/docker/router.conf @@ -46,7 +46,7 @@ command=java -Ddruid.client.https.keyManagerPassword=druid123 -Ddruid.client.https.keyStorePassword=druid123 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml - -cp /shared/docker/lib/* + -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar org.apache.druid.cli.Main server router redirect_stderr=true priority=100 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
