Repository: yetus Updated Branches: refs/heads/master 51166fdfa -> 2b9f425ab
YETUS-534. Provide a way to set snappy and isal location in hadoop personality Signed-off-by: Sean Busbey <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/0d6f7d3a Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/0d6f7d3a Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/0d6f7d3a Branch: refs/heads/master Commit: 0d6f7d3a2d5af96f32883bddaa55e8aae59dd308 Parents: 51166fd Author: Allen Wittenauer <[email protected]> Authored: Wed Jul 26 18:08:20 2017 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Fri Aug 18 12:16:58 2017 -0700 ---------------------------------------------------------------------- .../in-progress/precommit-advanced.md | 12 +++ precommit/personality/hadoop.sh | 86 ++++++++++++++++---- precommit/test-patch.sh | 4 + 3 files changed, 84 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/0d6f7d3a/asf-site-src/source/documentation/in-progress/precommit-advanced.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-advanced.md b/asf-site-src/source/documentation/in-progress/precommit-advanced.md index 7bdb8db..264f273 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-advanced.md +++ b/asf-site-src/source/documentation/in-progress/precommit-advanced.md @@ -163,6 +163,18 @@ function personality_globals } ``` +Additionally, a personality may require some outside help from the user. The `personality_parse_args` +function is called almost immediately after the personality is loaded and plug-ins parse arguments. + +```bash +function personality_parse_args +{ + echo "$*" +} +``` + +It is important to note that this function is called AFTER personality_globals. + ## Test Determination The `personality_file_tests` function determines which tests to turn on based upon the file name. It is relatively simple. For example, to turn on a full suite of tests for Java files: http://git-wip-us.apache.org/repos/asf/yetus/blob/0d6f7d3a/precommit/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh index a26576a..1d7d696 100755 --- a/precommit/personality/hadoop.sh +++ b/precommit/personality/hadoop.sh @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# +# SHELLDOC-IGNORE +# # Override these to match Apache Hadoop's requirements personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" @@ -42,6 +45,25 @@ function personality_globals fi } +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --hadoop-isal-prefix=*) + ISAL_HOME=${i#*=} + ;; + --hadoop-openssl-prefix=*) + OPENSSL_HOME=${i#*=} + ;; + --hadoop-snappy-prefix=*) + SNAPPY_HOME=${i#*=} + ;; + esac + done +} + ## @description Calculate the actual module ordering ## @audience private ## @stability evolving @@ -143,11 +165,13 @@ function yarn_ui2_flag ## @stability evolving function hadoop_native_flags { - if [[ ${BUILD_NATIVE} != true ]]; then return fi + declare -a args + declare windows=false + # Based upon HADOOP-11937 # # Some notes: @@ -160,34 +184,60 @@ function hadoop_native_flags # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 # + args=("-Drequire.test.libhadoop") + + if [[ -d "${ISAL_HOME}/include" ]]; then + args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") + fi + + if [[ -d "${OPENSSL_HOME}/include" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") + fi + + if [[ -d "${SNAPPY_HOME}/include" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") + fi + case ${OSTYPE} in Linux) # shellcheck disable=SC2086 - echo -Pnative -Drequire.libwebhdfs \ - -Drequire.snappy -Drequire.openssl -Drequire.fuse \ - -Drequire.test.libhadoop + echo \ + -Pnative -Drequire.snappy -Drequire.openssl -Drequire.fuse \ + "${args[@]}" ;; Darwin) - JANSSON_INCLUDE_DIR="${HADOOP_HOMEBREW_DIR}/opt/jansson/include" - JANSSON_LIBRARY="${HADOOP_HOMEBREW_DIR}/opt/jansson/lib" - export JANSSON_LIBRARY JANSSON_INCLUDE_DIR - # shellcheck disable=SC2086 echo \ - -Pnative -Drequire.snappy \ - -Drequire.openssl \ - -Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/ \ - -Dopenssl.include=${HADOOP_HOMEBREW_DIR}/opt/openssl/include \ - -Dopenssl.lib=${HADOOP_HOMEBREW_DIR}/opt/openssl/lib \ - -Drequire.libwebhdfs -Drequire.test.libhadoop + "${args[@]}" \ + -Pnative \ + -Drequire.snappy \ + -Drequire.openssl \ + -Dopenssl.prefix="${HADOOP_HOMEBREW_DIR}/opt/openssl/" \ + -Dopenssl.include="${HADOOP_HOMEBREW_DIR}/opt/openssl/include" \ + -Dopenssl.lib="${HADOOP_HOMEBREW_DIR}/opt/openssl/lib" + ;; + Windows_NT) + windows=true + ;; + CYGWIN*) + windows=true + ;; + MINGW32*) + windows=true + ;; + MSYS*) + windows=true ;; *) - # shellcheck disable=SC2086 echo \ - -Pnative \ - -Drequire.snappy -Drequire.openssl \ - -Drequire.test.libhadoop + "${args[@]}" ;; esac + + if [[ ${windows} = true ]]; then + echo \ + "${args[@]}" \ + -Drequire.snappy -Drequire.openssl -Pnative-win + fi } ## @description Queue up modules for this personality http://git-wip-us.apache.org/repos/asf/yetus/blob/0d6f7d3a/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index aef362a..3eab2ff 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -2996,6 +2996,10 @@ function initialize parse_args_plugins "$@" + if declare -f personality_parse_args >/dev/null; then + personality_parse_args "$@" + fi + BUGCOMMENTS=${BUGCOMMENTS:-${BUGSYSTEMS}} if [[ ! ${BUGCOMMENTS} =~ console ]]; then BUGCOMMENTS="${BUGCOMMENTS} console"
