abhishekrb19 commented on code in PR #13877:
URL: https://github.com/apache/druid/pull/13877#discussion_r1124936963
##########
integration-tests-ex/cases/cluster.sh:
##########
@@ -27,15 +27,16 @@ set -e
# Enable for debugging
#set -x
-export MODULE_DIR=$(cd $(dirname $0) && pwd)
+export BASE_MODULE_DIR=$(cd $(dirname $0) && pwd)
+
+# The location of the tests, if different than the
Review Comment:
nit: incomplete comment
##########
it.sh:
##########
@@ -41,22 +41,27 @@ Usage: $0 cmd [category]
Build druid-it-tools
image
Build the test image
- up <category>
+ up <category> [<module>]
Start the cluster for category
- down <category>
+ down <category> [<module>]
Stop the cluster for category
- test <category>
+ test <category> [<module>]
Start the cluster, run the test for category, and stop the cluster
- tail <category>
+ tail <category> [<module>]
Show the last 20 lines of each container log
- gen
+ gen <category> [<module>]
Generate docker-compose.yaml files (done automatically on up)
run one IT in Travis (build dist, image, run test, tail logs)
- github <category>
+ github <category> [<module>]
Run one IT in Github Workflows (run test, tail logs)
prune
prune Docker volumes
+Arguments:
+ category: A defined IT JUnit category, and IT-<category> profile
Review Comment:
nice!
##########
integration-tests-ex/cases/pom.xml:
##########
@@ -21,517 +21,477 @@
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
- <modelVersion>4.0.0</modelVersion>
+ <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.druid.integration-tests</groupId>
- <artifactId>druid-it-cases</artifactId>
- <name>druid-it-cases</name>
- <description>New Integration Tests</description>
+ <groupId>org.apache.druid.integration-tests</groupId>
+ <artifactId>druid-it-cases</artifactId>
+ <name>druid-it-cases</name>
+ <description>New Integration Tests</description>
- <parent>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid</artifactId>
- <version>26.0.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid</artifactId>
+ <version>26.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
- <properties>
- <it.category>MustSetACategory</it.category>
- </properties>
+ <properties>
+ <it.category>MustSetACategory</it.category>
+ </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-integration-tests</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <!-- See
https://maven.apache.org/plugins/maven-jar-plugin/examples/create-test-jar.html
-->
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-integration-tests</artifactId>
- <version>${project.parent.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-services</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-indexing-service</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-multibindings</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-yaml</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.docker-java</groupId>
- <artifactId>docker-java-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.docker-java</groupId>
- <artifactId>docker-java-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.docker-java</groupId>
- <artifactId>docker-java-transport-netty</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.docker-java</groupId>
- <artifactId>docker-java</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.github.docker-java</groupId>
- <artifactId>docker-java-transport-jersey</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-kqueue</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-core</artifactId>
- <version>${aws.sdk.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.api-client</groupId>
- <artifactId>google-api-client</artifactId>
- <version>${com.google.apis.client.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-storage</artifactId>
- <version>${com.google.apis.storage.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.api-client</groupId>
- <artifactId>google-api-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-storage</artifactId>
- <version>8.6.0</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client</artifactId>
- <version>${com.google.apis.client.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client-jackson2</artifactId>
- <version>${com.google.apis.client.version}</version>
- <scope>provided</scope>
- </dependency>
- <!-- Tests can choose either the MySQL or MariaDB driver. -->
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.version}</version>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.mariadb.jdbc</groupId>
- <artifactId>mariadb-java-client</artifactId>
- <version>${mariadb.version}</version>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.jdbi</groupId>
- <artifactId>jdbi</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>mysql-metadata-storage</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-azure-extensions</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <version>${aws.sdk.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-s3-extensions</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-hdfs-storage</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-google-extensions</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-gcp-common</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-integration-tests</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <!-- See
https://maven.apache.org/plugins/maven-jar-plugin/examples/create-test-jar.html
-->
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-integration-tests</artifactId>
+ <version>${project.parent.version}</version>
+ <classifier>tests</classifier>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-processing</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-server</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-services</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-indexing-service</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-multibindings</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java-transport-netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java-transport-jersey</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-kqueue</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-storage</artifactId>
+ <version>${com.google.apis.storage.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-storage</artifactId>
+ <version>8.6.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-jackson2</artifactId>
+ </dependency>
+ <!-- Tests can choose either the MySQL or MariaDB driver. -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>${mariadb.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jdbi</groupId>
+ <artifactId>jdbi</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>mysql-metadata-storage</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-azure-extensions</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-s3-extensions</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-hdfs-storage</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-google-extensions</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-gcp-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-sql</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-multi-stage-query</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-catalog</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${apache.kafka.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>pl.pragmatists</groupId>
- <artifactId>JUnitParams</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>5.4.0</version>
- </dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.apache.druid</groupId>
+ <artifactId>druid-sql</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-multi-stage-query</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-catalog</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${apache.kafka.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>pl.pragmatists</groupId>
+ <artifactId>JUnitParams</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </dependency>
+ </dependencies>
- <!-- Exclude ITs from surefire. Required because they end with "Test". -->
- <build>
+ <!-- Exclude ITs from surefire. Required because they end with "Test". -->
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/IT*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <!--
+ ~ The analysis gets confused between javax.xml.bind:jaxb-api and
jakarta.xml.bind:jakarta.xml.bind-api.
+ ~ The latter is a transitive dependency of jackson 2.10+.
+ -->
+ <ignoredUsedUndeclaredDependencies>
+
<ignoredUsedUndeclaredDependency>org.glassfish.hk2.external:jakarta.inject</ignoredUsedUndeclaredDependency>
+ </ignoredUsedUndeclaredDependencies>
+ <!-- Dynamically loaded. -->
+ <ignoredUnusedDeclaredDependencies>
+
<ignoredUnusedDeclaredDependency>mysql:mysql-connector-java:jar</ignoredUnusedDeclaredDependency>
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.owasp</groupId>
+ <artifactId>dependency-check-maven</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>IT-HighAvailability</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>HighAvailability</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-BatchIndex</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>BatchIndex</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-InputSource</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>InputSource</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-InputFormat</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>InputFormat</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-AzureDeepStorage</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>AzureDeepStorage</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-MultiStageQuery</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>MultiStageQuery</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-MultiStageQueryWithMM</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>MultiStageQueryWithMM</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-Catalog</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>Catalog</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-S3DeepStorage</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>S3DeepStorage</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <id>IT-GcsDeepStorage</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <it.category>GcsDeepStorage</it.category>
+ </properties>
+ </profile>
+ <profile>
+ <!-- Run the integration tests in Docker. Assumes that the cluster has
been
+ started via $DRUID_DEV/it.sh up <Category> or $DRUID_DEV/it.sh test
<Category>
+ Cluster start/stop is not done here to avoid duplicating
functionality.
+ Builds should invoke ITs using it.sh, not directy using Maven.
+ -->
+ <id>docker-tests</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>**/IT*</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <!--
- ~ The analysis gets confused between
javax.xml.bind:jaxb-api and jakarta.xml.bind:jakarta.xml.bind-api.
- ~ The latter is a transitive dependency of jackson 2.10+.
- -->
- <ignoredUsedUndeclaredDependencies>
-
<ignoredUsedUndeclaredDependency>org.glassfish.hk2.external:jakarta.inject</ignoredUsedUndeclaredDependency>
- </ignoredUsedUndeclaredDependencies>
- <!-- Dynamically loaded. -->
- <ignoredUnusedDeclaredDependencies>
-
<ignoredUnusedDeclaredDependency>mysql:mysql-connector-java:jar</ignoredUnusedDeclaredDependency>
- </ignoredUnusedDeclaredDependencies>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.owasp</groupId>
- <artifactId>dependency-check-maven</artifactId>
+ <plugin>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>3.0.0-M7</version>
+ <dependencies>
+ <!-- Required to force Failsafe to use JUnit instead of TestNG.
+ junit47 is required to use test categories. -->
+ <dependency>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>surefire-junit47</artifactId>
+ <version>3.0.0-M7</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>functional-test</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
<configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
+ <!-- Turning on logs so that travis does not time out tests
for not providing any output. -->
Review Comment:
Is this comment (and hack) still valid since we no longer use Travis?
##########
integration-tests-ex/cases/cluster/template.py:
##########
@@ -49,16 +49,21 @@ def generate(template_path, template):
'''
# Compute the cluster (test category) name from the template path which
- # we assume to be module/<something>/<template>/<something>.py
+ # we assume to be <module>/templates/<something>.py
template_path = Path(template_path)
- cluster = template_path.stem
+ cluster = template_path.parent.name
- # Move up to the module (that is, the cases folder) relative to the
template file.
- module_dir = Path(__file__).parent.parent
+ # Move up to the module relative to the template file.
+ module_dir = template_path.parent.parent.parent
# The target location for the output file is
<module>/target/cluster/<cluster>/docker-compose.yaml
target_dir = module_dir.joinpath("target")
+ os.makedirs(target_dir, exist_ok=True)
target_file = target_dir.joinpath('cluster', cluster,
'docker-compose.yaml')
+ # Uncomment the following for debugging
+ #print("template_path:", template_path)
Review Comment:
Perhaps we could use the `logging` library and add these to `DEBUG` logs
rather than comment-uncomment.
##########
integration-tests-ex/docs/docker.md:
##########
@@ -211,6 +211,37 @@ when it starts. If you start, then restart the MySQL
container, you *must*
remove the `db` directory before restart or MySQL will fail due to existing
files.
+### Per-test Extensions
+
+The image build includes a standard set of extensions. Contrib or custom
extensions
+may wish to add additional extensions. This is most easily done not by
altering the
+image, but by adding the extensions at cluster startup. If the shared
directory has
+an `extensions` subdirectory, then that directory is added to the extension
search
+path on container startup. To add an extension `my-extension`, your shared
directory
+should look like this:
+
+```text
+shared
++- ...
++- extensions
+ +- my-extension
+ +- my-extension-<version>.jar
++- ...
+```
+
+The the `extensions` directory should be created within the per-cluster
`setup.sh` script
Review Comment:
nit: "the" repeated twice
##########
docs/configuration/index.md:
##########
@@ -122,6 +122,7 @@ Many of Druid's external dependencies can be plugged in as
modules. Extensions c
|`druid.extensions.useExtensionClassloaderFirst`|This is a boolean flag that
determines if Druid extensions should prefer loading classes from their own
jars rather than jars bundled with Druid. If false, extensions must be
compatible with classes provided by any jars bundled with Druid. If true,
extensions may depend on conflicting versions.|false|
|`druid.extensions.hadoopContainerDruidClasspath`|Hadoop Indexing launches
hadoop jobs and this configuration provides way to explicitly set the user
classpath for the hadoop job. By default this is computed automatically by
druid based on the druid process classpath and set of extensions. However,
sometimes you might want to be explicit to resolve dependency conflicts between
druid and hadoop.|null|
|`druid.extensions.addExtensionsToHadoopContainer`|Only applicable if
`druid.extensions.hadoopContainerDruidClasspath` is provided. If set to true,
then extensions specified in the loadList are added to hadoop container
classpath. Note that when `druid.extensions.hadoopContainerDruidClasspath` is
not provided then extensions are always added to hadoop container
classpath.|false|
+|`druid.extensions.path`|An optional array of paths to search for extensions.
The `directory` entry acts as the first entry in the search path. Primarily for
testing and Docker-based environments.|null (i.e. no additional search path)|
Review Comment:
Since this setting can hold an array of paths, should we suffix it
`pathList` or `paths`?
##########
processing/src/main/java/org/apache/druid/guice/ExtensionsConfig.java:
##########
@@ -20,35 +20,63 @@
package org.apache.druid.guice;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
+import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;
+
import java.util.LinkedHashSet;
+import java.util.List;
/**
+ * Configuration for Druid extensions.
*/
public class ExtensionsConfig
{
public static final String PROPERTY_BASE = "druid.extensions";
+ public static final String DEFAULT_EXTENSIONS_DIR = "extensions";
@JsonProperty
@NotNull
private boolean searchCurrentClassloader = true;
+ /**
+ * The location of the "primary" extensions directory. If the path is
relative
+ * (as in the default), then the path is relative to the Druid directory
+ * (assuming the Druid product directory is indicated by the working
directory.)
+ * <p>
+ * The value can be blank to indicate to ignore this value and only use the
path.
+ * (If the value is omitted from the config file, it defaults to
"extensions", so
+ * we need a blank value to say to ignore this property.)
+ */
+ @JsonProperty
+ private String directory = DEFAULT_EXTENSIONS_DIR;
+
+ /**
+ * Extensions path. Items may be relative to the Druid home directory, or
absolute.
+ * Extensions are resolved in order along the path: the first match wins. If
+ * {@link #directory} is set, it becomes the first element on the full path.
+ * A typical use is to let {@code directory} point to {@code
$DRUID_HOME/extensions},
+ * and use {@link path} to point to extra extensions mounted into a Docker
container.
+ * Primarily for testing.
+ */
@JsonProperty
- private String directory = "extensions";
+ @Nullable
+ private List<String> path;
Review Comment:
Please see my related comment on calling this `paths` or `pathList`. It's
not apparent to me without looking at its type
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
