This is an automated email from the ASF dual-hosted git repository. agozhiy pushed a commit to branch resurrection in repository https://gitbox.apache.org/repos/asf/drill-test-framework.git
commit b4e0895bbd9c8204d3c1e19bf2531f0b4d490346 Author: Anton Gozhiy <[email protected]> AuthorDate: Fri Feb 11 19:52:09 2022 +0200 Added docker maven plugin, changed defaults to make it build and run easier --- bin/run_tests | 12 ++- framework/pom.xml | 85 ++++++++++++++++++++-- .../drill/test/framework/DrillQueryProfile.java | 10 +-- .../framework/DrillTestFrameworkUnitTests.java | 4 +- 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index 0186f96c..ec5390dc 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -5,6 +5,7 @@ source conf/drillTestConfig.properties cd ${bin}/../ default_debug_string="-Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,address=50000,server=y,suspend=y" +JDBC_DRIVER_CP="${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar" ARGS=() while [[ $# > 0 ]] @@ -20,6 +21,11 @@ do DRILL_TEST_FRAMEWORK_JAVA_OPTS="$DRILL_TEST_FRAMEWORK_JAVA_OPTS $default_debug_string" shift ;; + --jdbcCP) + shift + JDBC_DRIVER_CP=$1 + shift + ;; *) ARGS+=("$1") shift @@ -31,10 +37,12 @@ done # and set ${JDBC_DRIVER_CP} inside conf/drillTestConfig.properties # java -cp conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar:${HADOOP_INSTALL_LOC}/lib/* org.apache.drill.test.framework.TestDriver $* +/sbin/ifconfig + # use the following line when testing apache drill JDBC driver if [[ $HADOOP_VERSION == *"mapr"* ]] then - java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}" + java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}" else - java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}" + java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}" fi diff --git a/framework/pom.xml b/framework/pom.xml index 018766ab..259ec077 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -9,10 +9,23 @@ <testng.version>6.4</testng.version> <log4j.version>1.2.17</log4j.version> <jackson.version>2.10.0</jackson.version> - <drill.version>${env.DRILL_VERSION}</drill.version> - <hadoop.version>${env.HADOOP_VERSION}</hadoop.version> - <runtimeDepenencies>${env.DRILL_HOME}/conf</runtimeDepenencies> + <drill.version>1.20.0-SNAPSHOT</drill.version> + <hadoop.version>3.2.2</hadoop.version> + <drill.test.framework.root>${project.basedir}/..</drill.test.framework.root> + <drill.home>${project.basedir}/../../drill</drill.home> + <drill.distribution>${drill.home}/distribution/target/apache-drill-${drill.version}/apache-drill-${drill.version}/</drill.distribution> + <drill.jdbc.driver.path>jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</drill.jdbc.driver.path> + <drill.java.exec.path>jars/drill-java-exec-${drill.version}.jar</drill.java.exec.path> + <m2.home>${env.HOME}/.m2</m2.home> + <runtimeDepenencies>${drill.distribution}/conf</runtimeDepenencies> <maven.surefire.plugin.version>2.19.1</maven.surefire.plugin.version> + <!-- Docker Plugin Properties --> + <maven.docker.plugin.version>0.38.1</maven.docker.plugin.version> + <docker.base.image>maven:3.8.4-jdk-11</docker.base.image> + <docker.automation.root>/root/testAutomation</docker.automation.root> + <docker.drill.test.framework.home>${docker.automation.root}/drill-test-framework</docker.drill.test.framework.home> + <docker.drill.home>${docker.automation.root}/apache-drill-${drill.version}</docker.drill.home> + <test.args>-s Functional -g empty</test.args> </properties> <dependencies> <dependency> @@ -92,7 +105,14 @@ <groupId>org.apache.drill.exec</groupId> <artifactId>drill-jdbc-all</artifactId> <version>${drill.version}</version> - <systemPath>${env.DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</systemPath> + <systemPath>${drill.distribution}/${drill.jdbc.driver.path}</systemPath> + <scope>system</scope> + </dependency> + <dependency> + <groupId>org.apache.drill.exec</groupId> + <artifactId>drill-java-exec</artifactId> + <version>${drill.version}</version> + <systemPath>${drill.distribution}/${drill.java.exec.path}</systemPath> <scope>system</scope> </dependency> <dependency> @@ -141,7 +161,7 @@ <id>mapr-releases</id> <url>http://repository.mapr.com/maven/</url> <releases> - <enabled>true</enabled> + <enabled>false</enabled> </releases> <snapshots> <enabled>false</enabled> @@ -150,6 +170,61 @@ </repositories> <build> <plugins> + <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>${maven.docker.plugin.version}</version> + <configuration> + <showLogs>true</showLogs> + <verbose>true</verbose> + <images> + <image> + <name>drill-test-framework</name> + <build> + <nocache>false</nocache> + <from>${docker.base.image}</from> + </build> + <run> + <containerNamePattern>%n_%t_%i</containerNamePattern> + <wait> + <exit>0</exit> + <time>1210000000</time> + </wait> + <volumes> + <bind> + <volume>${drill.test.framework.root}:${docker.drill.test.framework.home}</volume> + <volume>${drill.distribution}:${docker.drill.home}</volume> + <volume>${m2.home}:/root/.m2</volume> + </bind> + </volumes> + <workingDir>${docker.drill.test.framework.home}</workingDir> + <network> + <mode>host</mode> + </network> + <cmd> + <shell>/bin/bash bin/run_tests --jdbcCP ${docker.drill.home}/${drill.jdbc.driver.path} ${test.args}</shell> + </cmd> + </run> + </image> + </images> + </configuration> + <!--<executions> + <execution> + <phase>install</phase> + <goals> + <goal>build</goal> + </goals> + </execution> + <execution> + <id>start</id> + <phase>test</phase> + <goals> + <goal>run</goal> + <goal>stop</goal> + </goals> + </execution> + </executions>--> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> diff --git a/framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java b/framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java index 83c5b959..2d255fd8 100644 --- a/framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java +++ b/framework/src/main/java/org/apache/drill/test/framework/DrillQueryProfile.java @@ -3,7 +3,7 @@ package org.apache.drill.test.framework; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import oadd.org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.server.rest.profile.CoreOperatorType; import java.util.List; import java.util.stream.Collectors; @@ -208,13 +208,13 @@ public class DrillQueryProfile { * @param operator * @return */ - public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) { + public long getOptimalMemoryPerOperator(final CoreOperatorType operator) { return this.fragmentProfiles .stream() .flatMap(f -> f.minorFragmentProfiles .stream() .flatMap(m -> m.operatorProfiles.stream()) - ).filter(o -> o.operatorId == operator.getNumber()) + ).filter(o -> o.operatorId == operator.getId()) .mapToLong(o -> o.optimalMemAllocation) .sum(); } @@ -223,14 +223,14 @@ public class DrillQueryProfile { * Get different operators in the profile. * @return a list of operators in the query profile. */ - public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() { + public List<CoreOperatorType> getOperatorsFromProfile() { return this.fragmentProfiles .stream().flatMap(f -> f.minorFragmentProfiles .stream() .flatMap(m -> m.operatorProfiles.stream()) ).mapToInt(o -> o.operatorId) .distinct() - .mapToObj(UserBitShared.CoreOperatorType::forNumber) + .mapToObj(CoreOperatorType::valueOf) .collect(Collectors.toList()); } diff --git a/framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java b/framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java index d7c39945..9ff2656f 100644 --- a/framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java +++ b/framework/src/test/java/org/apache/drill/test/framework/DrillTestFrameworkUnitTests.java @@ -2,7 +2,7 @@ package org.apache.drill.test.framework; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; -import oadd.org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.server.rest.profile.CoreOperatorType; import org.apache.drill.test.framework.common.DrillJavaTestBase; import org.apache.log4j.Logger; import org.testng.Assert; @@ -182,7 +182,7 @@ public class DrillTestFrameworkUnitTests extends DrillJavaTestBase { LOG.info("Memory estimated by RM planner: " + rmMemEstimate); Assert.assertTrue(rmMemEstimate > 0, "RM estimated memory should be greater than 0"); - List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile(); + List<CoreOperatorType> operators = profile.getOperatorsFromProfile(); Assert.assertTrue(operators.size() > 0, "Number of operators in the profile should be greater than 0"); operators.forEach(LOG::info);
