imay closed pull request #480: Add md5sum check for third party packages
URL: https://github.com/apache/incubator-doris/pull/480
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 8a8e436a..fa5510cb 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -48,10 +48,35 @@ fi
mkdir -p ${TP_DIR}/src
-download() {
+md5sum_bin=md5sum
+if ! command -v ${md5sum_bin} >/dev/null 2>&1; then
+ echo "Warn: md5sum is not installed"
+ md5sum_bin=""
+fi
+
+md5sum_func() {
+ local FILENAME=$1
+ local DESC_DIR=$2
+ local MD5SUM=$3
+
+ if [ "$md5sum_bin" == "" ]; then
+ return 0
+ else
+ md5=`md5sum "$DESC_DIR/$FILENAME"`
+ if [ "$md5" != "$MD5SUM $DESC_DIR/$FILENAME" ]; then
+ echo "$DESC_DIR/$FILENAME md5sum check failed!"
+ return 1
+ fi
+ fi
+
+ return 0
+}
+
+download_func() {
local FILENAME=$1
local DOWNLOAD_URL=$2
local DESC_DIR=$3
+ local MD5SUM=$4
if [ -z "$FILENAME" ]; then
echo "Error: No file name specified to download"
@@ -70,16 +95,24 @@ download() {
SUCCESS=0
for attemp in 1 2; do
if [ -r "$DESC_DIR/$FILENAME" ]; then
- echo "Archive $FILENAME already exist."
- SUCCESS=1
- break;
+ if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
+ echo "Archive $FILENAME already exist."
+ SUCCESS=1
+ break;
+ fi
+ echo "Archive $FILENAME will be removed and download again."
+ rm -f "$DESC_DIR/$FILENAME"
else
echo "Downloading $FILENAME from $DOWNLOAD_URL to $DESC_DIR"
wget --no-check-certificate $DOWNLOAD_URL -O $DESC_DIR/$FILENAME
if [ "$?"x == "0"x ]; then
- SUCCESS=1
- echo "Success to download $FILENAME"
- break;
+ if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
+ SUCCESS=1
+ echo "Success to download $FILENAME"
+ break;
+ fi
+ echo "Archive $FILENAME will be removed and download again."
+ rm -f "$DESC_DIR/$FILENAME"
else
echo "Failed to download $FILENAME. attemp: $attemp"
fi
@@ -97,12 +130,13 @@ echo "===== Downloading thirdparty archives..."
for TP_ARCH in ${TP_ARCHIVES[*]}
do
NAME=$TP_ARCH"_NAME"
+ MD5SUM=$TP_ARCH"_MD5SUM"
if test "x$REPOSITORY_URL" = x; then
URL=$TP_ARCH"_DOWNLOAD"
- download ${!NAME} ${!URL} $TP_SOURCE_DIR
+ download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM}
else
URL="${REPOSITORY_URL}/${!NAME}"
- download ${!NAME} ${URL} $TP_SOURCE_DIR
+ download_func ${!NAME} ${URL} $TP_SOURCE_DIR ${!MD5SUM}
fi
done
echo "===== Downloading thirdparty archives...done"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index d5483053..e8894cd0 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -58,132 +58,158 @@ export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
LIBEVENT_DOWNLOAD="http://palo-opensource.gz.bcebos.com/libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip?authorization=bce-auth-v1%2F069fc2786e464e63a5f1183824ddb522%2F2018-06-22T02%3A37%3A48Z%2F-1%2Fhost%2F031b0cc42ab83ca4e0ec3608cba963e95c1ddc46fc70a14457323e2d7960e6ef";
LIBEVENT_NAME=libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip
LIBEVENT_SOURCE=libevent-master
+LIBEVENT_MD5SUM="e8b9ba50270ba3b520aec8ff1089f9d7"
# openssl
OPENSSL_DOWNLOAD="https://www.openssl.org/source/openssl-1.0.2k.tar.gz";
OPENSSL_NAME=openssl-1.0.2k.tar.gz
OPENSSL_SOURCE=openssl-1.0.2k
+OPENSSL_MD5SUM="f965fc0bf01bf882b31314b61391ae65"
# thrift
THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz";
THRIFT_NAME=thrift-0.9.3.tar.gz
THRIFT_SOURCE=thrift-0.9.3
+THRIFT_MD5SUM="88d667a8ae870d5adeca8cb7d6795442"
# llvm
LLVM_DOWNLOAD="http://releases.llvm.org/3.4.2/llvm-3.4.2.src.tar.gz";
LLVM_NAME=llvm-3.4.2.src.tar.gz
LLVM_SOURCE=llvm-3.4.2.src
+LLVM_MD5SUM="a20669f75967440de949ac3b1bad439c"
# clang
CLANG_DOWNLOAD="http://releases.llvm.org/3.4.2/cfe-3.4.2.src.tar.gz";
CLANG_NAME=cfe-3.4.2.src.tar.gz
CLANG_SOURCE=cfe-3.4.2.src
+CLANG_MD5SUM="87945973b7c73038871c5f849a818588"
# compiler-rt
COMPILER_RT_DOWNLOAD="http://releases.llvm.org/5.0.0/compiler-rt-5.0.0.src.tar.xz";
COMPILER_RT_NAME=compiler-rt-5.0.0.src.tar.xz
COMPILER_RT_SOURCE=compiler-rt-5.0.0.src
+COMPILER_RT_MD5SUM="da735894133589cbc6052c8ef06b1230"
# protobuf
PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.5.1.tar.gz";
PROTOBUF_NAME=protobuf-3.5.1.tar.gz
PROTOBUF_SOURCE=protobuf-3.5.1
+PROTOBUF_MD5SUM="710f1a75983092c9b45ecef207236104"
# gflags
GFLAGS_DOWNLOAD="https://github.com/gflags/gflags/archive/v2.2.0.tar.gz";
GFLAGS_NAME=gflags-2.2.0.tar.gz
GFLAGS_SOURCE=gflags-2.2.0
+GFLAGS_MD5SUM="b99048d9ab82d8c56e876fb1456c285e"
# glog
GLOG_DOWNLOAD="https://github.com/google/glog/archive/v0.3.3.tar.gz";
GLOG_NAME=glog-0.3.3.tar.gz
GLOG_SOURCE=glog-0.3.3
+GLOG_MD5SUM="c1f86af27bd9c73186730aa957607ed0"
# gtest
GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.8.0.tar.gz";
GTEST_NAME=googletest-release-1.8.0.tar.gz
GTEST_SOURCE=googletest-release-1.8.0
+GTEST_MD5SUM="16877098823401d1bf2ed7891d7dce36"
# snappy
SNAPPY_DOWNLOAD="https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz";
SNAPPY_NAME=snappy-1.1.4.tar.gz
SNAPPY_SOURCE=snappy-1.1.4
+SNAPPY_MD5SUM="c328993b68afe3e5bd87c8ea9bdeb028"
# gperftools
GPERFTOOLS_DOWNLOAD="https://github.com/gperftools/gperftools/archive/gperftools-2.7.tar.gz";
GPERFTOOLS_NAME=gperftools-2.7.tar.gz
GPERFTOOLS_SOURCE=gperftools-gperftools-2.7
+GPERFTOOLS_MD5SUM="797e7b7f6663288e2b90ab664861c61a"
# zlib
ZLIB_DOWNLOAD="https://sourceforge.net/projects/libpng/files/zlib/1.2.11/zlib-1.2.11.tar.gz";
ZLIB_NAME=zlib-1.2.11.tar.gz
ZLIB_SOURCE=zlib-1.2.11
+ZLIB_MD5SUM="1c9f62f0778697a09d36121ead88e08e"
# lz4
LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.7.5.tar.gz";
LZ4_NAME=lz4-1.7.5.tar.gz
LZ4_SOURCE=lz4-1.7.5
+LZ4_MD5SUM="c9610c5ce97eb431dddddf0073d919b9"
# bzip
# BZIP_DOWNLOAD="http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz";
BZIP_DOWNLOAD="https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz";
BZIP_NAME=bzip2-1.0.6.tar.gz
BZIP_SOURCE=bzip2-1.0.6
+BZIP_MD5SUM="00b516f4704d4a7cb50a1d97e6e8e15b"
# lzo2
LZO2_DOWNLOAD="https://github.com/damageboy/lzo2/archive/master.zip";
LZO2_NAME=lzo2-master.zip
LZO2_SOURCE=lzo2-master
+LZO2_MD5SUM="0afc2456c8f425778d8a4e976f97d717"
# rapidjson
RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz";
RAPIDJSON_NAME=rapidjson-1.1.0.tar.gz
RAPIDJSON_SOURCE=rapidjson-1.1.0
+RAPIDJSON_MD5SUM="badd12c511e081fec6c89c43a7027bce"
# curl
CURL_DOWNLOAD="https://curl.haxx.se/download/curl-7.54.0.tar.gz";
CURL_NAME=curl-7.54.0.tar.gz
CURL_SOURCE=curl-7.54.0
+CURL_MD5SUM="18091896d871982cc4c2b307885eacb3"
# RE2
RE2_DOWNLOAD="https://github.com/google/re2/archive/2017-05-01.tar.gz";
RE2_NAME=re2-2017-05-01.tar.gz
RE2_SOURCE=re2-2017-05-01
+RE2_MD5SUM="4aa65a0b22edacb7ddcd7e4aec038dcf"
# boost
BOOST_DOWNLOAD="https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz";
BOOST_NAME=boost_1_64_0.tar.gz
BOOST_SOURCE=boost_1_64_0
+BOOST_MD5SUM="319c6ffbbeccc366f14bb68767a6db79"
# mysql
MYSQL_DOWNLOAD="https://github.com/mysql/mysql-server/archive/mysql-5.7.18.tar.gz";
MYSQL_NAME=mysql-5.7.18.tar.gz
MYSQL_SOURCE=mysql-server-mysql-5.7.18
+MYSQL_MD5SUM="58598b10dce180e4d1fbdd7cf5fa68d6"
# boost for mysql
BOOST_FOR_MYSQL_DOWNLOAD="http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz";
BOOST_FOR_MYSQL_NAME=boost_1_59_0.tar.gz
BOOST_FOR_MYSQL_SOURCE=boost_1_59_0
+BOOST_FOR_MYSQL_MD5SUM="51528a0e3b33d9e10aaa311d9eb451e3"
# leveldb
LEVELDB_DOWNLOAD="https://github.com/google/leveldb/archive/v1.20.tar.gz";
LEVELDB_NAME=leveldb-1.20.tar.gz
LEVELDB_SOURCE=leveldb-1.20
+LEVELDB_MD5SUM="298b5bddf12c675d6345784261302252"
# brpc
BRPC_DOWNLOAD="https://github.com/brpc/brpc/archive/v0.9.0.tar.gz";
BRPC_NAME=brpc-0.9.0.tar.gz
BRPC_SOURCE=brpc-0.9.0
+BRPC_MD5SUM="2e79f413614b99d44083499a81c0db67"
# rocksdb
ROCKSDB_DOWNLOAD="https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz";
ROCKSDB_NAME=rocksdb-5.14.2.tar.gz
ROCKSDB_SOURCE=rocksdb-5.14.2
+ROCKSDB_MD5SUM="b72720ea3b1e9ca9e4ed0febfef65b14"
# librdkafka
LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/v0.11.6-RC5.tar.gz";
LIBRDKAFKA_NAME=librdkafka-0.11.6-RC5.tar.gz
LIBRDKAFKA_SOURCE=librdkafka-0.11.6-RC5
+LIBRDKAFKA_MD5SUM="2e4ecef2df277e55a0144eb6d185e18a"
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES="LIBEVENT OPENSSL THRIFT LLVM CLANG COMPILER_RT PROTOBUF
GFLAGS GLOG GTEST RAPIDJSON SNAPPY GPERFTOOLS ZLIB LZ4 BZIP LZO2 CURL RE2 BOOST
MYSQL BOOST_FOR_MYSQL LEVELDB BRPC ROCKSDB LIBRDKAFKA"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
