Repository: incubator-impala Updated Branches: refs/heads/branch-2.7.0 53439d3d8 -> 6e119a036
IMPALA-4171: Remove JAR from repo. By ASF rules, we can't have JARs in releases. The releases are just tarballs of the repo. This patch removes from the repo the single JAR there, which was a version of a JAR that is built during data load, with one string changed. The JAR is used only for testing. Instead of building that jar with the different string and saving the result in git, daa loading will now build the jar twice, with one Java source file slightly changed. Change-Id: Icee7b8c32b08e064dea4a14624acff6021ef5ce1 Reviewed-on: http://gerrit.cloudera.org:8080/4499 Reviewed-by: Alex Behm <alex.b...@cloudera.com> Tested-by: Internal Jenkins Conflicts: testdata/bin/copy-udfs-udas.sh Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6e119a03 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6e119a03 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6e119a03 Branch: refs/heads/branch-2.7.0 Commit: 6e119a0365ae7a5c9c6666a5aae9a87154e461ce Parents: 53439d3 Author: Jim Apple <jbap...@cloudera.com> Authored: Tue Sep 20 13:54:55 2016 -0700 Committer: Jim Apple <jbap...@cloudera.com> Committed: Thu Sep 22 03:13:28 2016 -0700 ---------------------------------------------------------------------- .gitignore | 3 + bin/check-rat-report.py | 3 +- testdata/bin/copy-udfs-udas.sh | 64 +++++++++++-------- testdata/udfs/README | 8 +-- testdata/udfs/impala-hive-udfs.jar | Bin 3801 -> 0 bytes .../java/com/cloudera/impala/TestUpdateUdf.java | 12 +--- 6 files changed, 47 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index ec52448..68a43cb 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ toolchain/ core core.* hs_err_pid*.log + +# Binaries disallowed by ASF rules +*.jar \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/bin/check-rat-report.py ---------------------------------------------------------------------- diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py index 06c822d..a2f3b52 100755 --- a/bin/check-rat-report.py +++ b/bin/check-rat-report.py @@ -67,7 +67,8 @@ for r in resources: break if not excluded: typename = r.findall('type')[0].attrib['name'] - if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']): + if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary'] + and clean_name[-4:] <> '.jar'): sys.stderr.write( "%s: %s\n" % ('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name)) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/testdata/bin/copy-udfs-udas.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh index 814973a..57c0bc7 100755 --- a/testdata/bin/copy-udfs-udas.sh +++ b/testdata/bin/copy-udfs-udas.sh @@ -26,7 +26,7 @@ if [ x${JAVA_HOME} == x ]; then echo JAVA_HOME not set exit 1 fi -. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 +. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1 BUILD=0 @@ -47,11 +47,19 @@ done if [ $BUILD -eq 1 ] then - pushd $IMPALA_HOME - make -j$CORES \ + pushd "${IMPALA_HOME}" + "${MAKE_CMD:-make}" "-j${IMPALA_BUILD_THREADS:-4}" \ TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir - cd $IMPALA_HOME/tests/test-hive-udfs - ${IMPALA_HOME}/bin/mvn-quiet.sh package + cd "${IMPALA_HOME}/tests/test-hive-udfs" + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" + # Change one of the Java files to make a new jar for testing + # purposes, then change it back + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'Old UDF'/'New UDF'/g '{}'" \; + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'New UDF'/'Old UDF'/g '{}'" \; popd fi @@ -63,27 +71,27 @@ fi # impala-hive-udfs.jar # test-udfs.ll # udf/uda samples (.so/.ll) -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO -hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/udf_test -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so -hadoop fs -put -f ${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar -hadoop fs -put -f ${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO" +hadoop fs -mkdir -p "${FILESYSTEM_PREFIX}/test-warehouse/udf_test" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so" +hadoop fs -put -f "${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar" +hadoop fs -put -f "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" echo "Done copying udf/uda libraries." http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/testdata/udfs/README ---------------------------------------------------------------------- diff --git a/testdata/udfs/README b/testdata/udfs/README index d8817bb..31f03b2 100644 --- a/testdata/udfs/README +++ b/testdata/udfs/README @@ -1,7 +1,5 @@ -Most of the test UDFs are generated as part of the build. This folder -contains prebuilt UDFs to facilitate tests that verify the UDF binary -can be upgraded. - -The jar is generated from $IMPALA_HOME/tests/test-hive-udfs. +Most of the test UDFs are generated as part of the build. After +copy-udfs-udas.sh is run, This folder contains prebuilt UDFs to +facilitate tests that verify the UDF binary can be upgraded. This is used by test_udfs.py. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/testdata/udfs/impala-hive-udfs.jar ---------------------------------------------------------------------- diff --git a/testdata/udfs/impala-hive-udfs.jar b/testdata/udfs/impala-hive-udfs.jar deleted file mode 100644 index d916937..0000000 Binary files a/testdata/udfs/impala-hive-udfs.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e119a03/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java ---------------------------------------------------------------------- diff --git a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java index f29e8af..2d76d73 100644 --- a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java +++ b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java @@ -21,8 +21,9 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.io.Text; /** - * Simple UDF for testing updating a UDF in HDFS. This file is built and - * the old version of the jar is saved to $IMPALA_HOME/testdata/udfs/ + * Simple UDF for testing updating a UDF in HDFS. This file is built + * twice, with one version of the jar having TestUpdateUdf.evaluate() + * return "Old UDF" and the other "New UDF". * * The build produces a new version of the UDF. The tests make sure the * jar can be updated without having to restart Impalad. @@ -32,13 +33,6 @@ public class TestUpdateUdf extends UDF { } public Text evaluate() { - return new Text("New UDF"); - } - - // Use this function instead of the other evaluate to regenerate the checked in jar. - /* - public Text evaluate() { return new Text("Old UDF"); } - */ }
