This is an automated email from the ASF dual-hosted git repository. todd pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7af981f7afdd25c6e0ed7e86c21bbc7710098a9e Author: Todd Lipcon <t...@apache.org> AuthorDate: Wed May 15 13:40:13 2019 -0700 fe: clean up POM and improve m2e integration This cleans up a few items in the FE pom: - removes old system properties like 'beeswax_port', 'impalad', and 'use_external_impalad' which have been dead code since 2013. - adds appropriate configuration for m2e to avoid some errors upon importing the project (configures various Maven plugins to ignore or execute via M2E) - sets up m2e to generate classes into target/eclipse-classes so that auto-rebuilds don't break running daemons (carrying over the same idea from the old eclipse:eclipse support) Additionally, this stops passing the be build directory to java.library.path via a system property, since that system property doesn't get forwarded into the Eclipse test run configurations. Instead, I added some code to NativeLibUtil to locate the be build directory and search the appropriate path explicitly. Change-Id: Ifdad9610d858d488eb95cb947ed123fe1ebfe62a Reviewed-on: http://gerrit.cloudera.org:8080/13365 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Todd Lipcon <t...@apache.org> --- fe/pom.xml | 84 +++++++++++----------- .../java/org/apache/impala/util/NativeLibUtil.java | 46 +++++++++--- impala-parent/pom.xml | 5 -- 3 files changed, 78 insertions(+), 57 deletions(-) diff --git a/fe/pom.xml b/fe/pom.xml index d6c9908..fdf3272 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -431,7 +431,12 @@ under the License. </plugins> </reporting> + <properties> + <buildOutputDirectory>${project.build.directory}/classes</buildOutputDirectory> + </properties> + <build> + <outputDirectory>${buildOutputDirectory}</outputDirectory> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -542,21 +547,7 @@ under the License. <trimStackTrace>false</trimStackTrace> <reportsDirectory>${surefire.reports.dir}</reportsDirectory> <redirectTestOutputToFile>true</redirectTestOutputToFile> - <argLine>-Djava.library.path=${java.library.path}:${backend.library.path} ${surefireJacocoArg}</argLine> - <systemProperties> - <property> - <name>testExecutionMode</name> - <value>${testExecutionMode}</value> - </property> - <property> - <name>beeswax_port</name> - <value>${beeswax_port}</value> - <name>impalad</name> - <value>${impalad}</value> - <name>use_external_impalad</name> - <value>${use_external_impalad}</value> - </property> - </systemProperties> + <argLine>${surefireJacocoArg}</argLine> </configuration> </plugin> @@ -588,31 +579,6 @@ under the License. </plugin> <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.4.0</version> - <executions> - <execution> - <goals> - <goal>java</goal> - </goals> - </execution> - </executions> - <configuration> - <systemProperties> - <systemProperty> - <key>java.library.path</key> - <value>${java.library.path}:${backend.library.path}</value> - </systemProperty> - <systemProperty> - <key>test.hive.testdata</key> - <value>${project.basedir}/../testdata/target/AllTypes.txt</value> - </systemProperty> - </systemProperties> - </configuration> - </plugin> - - <plugin> <groupId>org.jacoco</groupId> <artifactId>jacoco-maven-plugin</artifactId> <version>0.7.6.201602180812</version> @@ -732,17 +698,34 @@ under the License. <versionRange>[2.0,)</versionRange> <goals> <goal>copy-dependencies</goal> + <goal>build-classpath</goal> </goals> </pluginExecutionFilter> <action> <ignore></ignore> </action> </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>de.jflex</groupId> + <artifactId>maven-jflex-plugin</artifactId> + <versionRange>[1.4.3,)</versionRange> + <goals> + <goal>generate</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute></execute> + </action> + </pluginExecution> </pluginExecutions> </lifecycleMappingMetadata> </configuration> </plugin> - <!-- mvn eclipse:eclipse generates Eclipse .project and .classpath files --> + <!-- mvn eclipse:eclipse generates Eclipse .project and .classpath files + NOTE: This is a deprecated Maven plugin. It's recommended to use + the native Eclipse "import Maven project" functionality (m2e) + instead --> <plugin> <groupId>org.apache.maven.plugins </groupId> <artifactId>maven-eclipse-plugin</artifactId> @@ -1125,6 +1108,25 @@ under the License. </plugins> </build> </profile> + + <!-- Profile which is automatically activated when building from + within Eclipse based on the presence of the m2e.version + property --> + <profile> + <id>eclipse-m2e</id> + <activation> + <property> + <name>m2e.version</name> + </property> + </activation> + <!-- By default, we separate Eclipse-built files from Maven-built + files. Otherwise, they are both in target/classes, and Eclipse + and Maven may clobber each other, complicating attaching to + a running process. --> + <properties> + <buildOutputDirectory>${project.build.directory}/${eclipse.output.directory}</buildOutputDirectory> + </properties> + </profile> </profiles> <dependencyManagement> diff --git a/fe/src/main/java/org/apache/impala/util/NativeLibUtil.java b/fe/src/main/java/org/apache/impala/util/NativeLibUtil.java index 877b36b..ceb1cf6 100644 --- a/fe/src/main/java/org/apache/impala/util/NativeLibUtil.java +++ b/fe/src/main/java/org/apache/impala/util/NativeLibUtil.java @@ -18,27 +18,51 @@ package org.apache.impala.util; import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; public class NativeLibUtil { + private final static Logger LOG = LoggerFactory.getLogger(NativeLibUtil.class); + /** - * Attempts to load the given library from all paths in java.libary.path. + * Attempts to load the given library from all paths in java.libary.path, + * as well as the current build directory (assuming we are in a test environment). * Throws a RuntimeException if the library was unable to be loaded from * any location. */ public static void loadLibrary(String libFileName) { - boolean found = false; - String javaLibPath = System.getProperty("java.library.path"); - for (String path: javaLibPath.split(":")) { + List<String> candidates = new ArrayList<>(Arrays.asList( + System.getProperty("java.library.path").split(":"))); + + // Fall back to automatically finding the library in test environments. + // This makes it easier to run tests from Eclipse without specially configuring + // the Run Configurations. + try { + String myPath = NativeLibUtil.class.getProtectionDomain() + .getCodeSource().getLocation().getPath(); + if (myPath.toString().endsWith("fe/target/classes/") || + myPath.toString().endsWith("fe/target/eclipse-classes/")) { + candidates.add(myPath + "../../../be/build/latest/service/"); + } + } catch (Exception e) { + LOG.warn("Unable to get path for NativeLibUtil class", e); + } + + for (String path: candidates) { File libFile = new File(path + File.separator + libFileName); if (libFile.exists()) { System.load(libFile.getPath()); - found = true; - break; + return; } } - if (!found) { - throw new RuntimeException("Failed to load " + libFileName + " from any " + - "location in java.library.path (" + javaLibPath + ")."); - } + + throw new RuntimeException("Failed to load " + libFileName + " from any " + + "candidate location:\n" + Joiner.on("\n").join(candidates)); } -} \ No newline at end of file +} diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml index 355bc05..62a6dc5 100644 --- a/impala-parent/pom.xml +++ b/impala-parent/pom.xml @@ -30,11 +30,6 @@ under the License. <jacoco.skip>true</jacoco.skip> <jacoco.data.file>${env.IMPALA_FE_TEST_COVERAGE_DIR}/jacoco.exec</jacoco.data.file> <jacoco.report.dir>${env.IMPALA_FE_TEST_COVERAGE_DIR}</jacoco.report.dir> - <test.hive.testdata>${project.basedir}/../testdata/target/AllTypes.txt</test.hive.testdata> - <backend.library.path>${env.IMPALA_HOME}/be/build/debug/service:${env.IMPALA_HOME}/be/build/release/service</backend.library.path> - <beeswax_port>21000</beeswax_port> - <impalad>localhost</impalad> - <testExecutionMode>reduced</testExecutionMode> <hadoop.version>${env.IMPALA_HADOOP_VERSION}</hadoop.version> <hive.version>${env.IMPALA_HIVE_VERSION}</hive.version> <hive.storage.api.version>2.3.0.${env.IMPALA_HIVE_VERSION}</hive.storage.api.version>