[thirdparty] allow different URL prefix This patch adds the option to allow different URL prefix when downloading third party dependencies.
Change-Id: I004828785d9b0d484806d743d7a1084a3ffd576b Reviewed-on: http://gerrit.cloudera.org:8080/11275 Reviewed-by: Adar Dembo <a...@cloudera.com> Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert <danburk...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/51519e65 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/51519e65 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/51519e65 Branch: refs/heads/master Commit: 51519e65bd3701b7549bc780c3d0fad45d799854 Parents: 4fe8967 Author: Hao Hao <hao....@cloudera.com> Authored: Thu Aug 16 12:50:43 2018 -0700 Committer: Hao Hao <hao....@cloudera.com> Committed: Thu Aug 23 23:18:21 2018 +0000 ---------------------------------------------------------------------- thirdparty/download-thirdparty.sh | 62 +++++++++++++++++++++++----------- thirdparty/vars.sh | 4 +++ 2 files changed, 47 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/51519e65/thirdparty/download-thirdparty.sh ---------------------------------------------------------------------- diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index e1389c1..7992065 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -55,17 +55,25 @@ unzip_to_source() { fetch_and_expand() { local FILENAME=$1 local SOURCE=$2 + local URL_PREFIX=$3 + if [ -z "$FILENAME" ]; then echo "Error: Must specify file to fetch" exit 1 fi + if [ -z "$URL_PREFIX" ]; then + echo "Error: Must specify url prefix to fetch" + exit 1 + fi + TAR_CMD=tar if [[ "$OSTYPE" == "darwin"* ]] && which gtar &>/dev/null; then TAR_CMD=gtar fi - FULL_URL="${CLOUDFRONT_URL_PREFIX}/${FILENAME}" + FULL_URL="${URL_PREFIX}/${FILENAME}" + SUCCESS=0 # Loop in case we encounter an error. for attempt in 1 2 3; do @@ -119,25 +127,41 @@ fetch_and_expand() { echo } +fetch_with_url_and_patch() { + local FILENAME=$1 + local SOURCE=$2 + local PATCH_LEVEL=$3 + local URL_PREFIX=$4 + # Remaining args are expected to be a list of patch commands + + delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL + if [ ! -d $SOURCE ]; then + fetch_and_expand $FILENAME $SOURCE $URL_PREFIX + pushd $SOURCE + shift 4 + # Run the patch commands + for f in "$@"; do + eval "$f" + done + touch patchlevel-$PATCH_LEVEL + popd + echo + fi +} + +# Call fetch_with_url_and_patch with the default dependency URL source. fetch_and_patch() { - local FILENAME=$1 - local SOURCE=$2 - local PATCH_LEVEL=$3 - # Remaining args are expected to be a list of patch commands - - delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL - if [ ! -d $SOURCE ]; then - fetch_and_expand $FILENAME $SOURCE - pushd $SOURCE - shift 3 - # Run the patch commands - for f in "$@"; do - eval "$f" - done - touch patchlevel-$PATCH_LEVEL - popd - echo - fi + local FILENAME=$1 + local SOURCE=$2 + local PATCH_LEVEL=$3 + + shift 3 + fetch_with_url_and_patch \ + $FILENAME \ + $SOURCE \ + $PATCH_LEVEL \ + $DEPENDENCY_URL \ + "$@" } mkdir -p $TP_SOURCE_DIR http://git-wip-us.apache.org/repos/asf/kudu/blob/51519e65/thirdparty/vars.sh ---------------------------------------------------------------------- diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 2804bb3..71beea6 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -30,6 +30,10 @@ TP_BUILD_DIR="$TP_DIR/build" # http://cloudera-thirdparty-libs.s3.amazonaws.com/ CLOUDFRONT_URL_PREFIX=http://d3dr9sfxru4sde.cloudfront.net +# Third party dependency downloading URL, default to the CloudFront +# Distribution URL. +DEPENDENCY_URL=${DEPENDENCY_URL:-$CLOUDFRONT_URL_PREFIX} + PREFIX_COMMON=$TP_DIR/installed/common PREFIX_DEPS=$TP_DIR/installed/uninstrumented PREFIX_DEPS_TSAN=$TP_DIR/installed/tsan
