This is an automated email from the ASF dual-hosted git repository. ztao1987 pushed a commit to branch taoz in repository https://gitbox.apache.org/repos/asf/hawq.git
The following commit(s) were added to refs/heads/taoz by this push: new 3922ae7 HAWQ-1799. include libhdfs3 in building path 3922ae7 is described below commit 3922ae775d87a928a6eefb3adecfa9c8d1c198f0 Author: ztao1987 <zhenglin.ta...@gmail.com> AuthorDate: Tue Aug 3 10:13:07 2021 +0800 HAWQ-1799. include libhdfs3 in building path --- .github/scripts/toolchain.sh | 106 +++++++++++++++++++++++++++++++++++++ GNUmakefile.in | 4 ++ config/c-compiler.m4 | 4 +- config/c-library.m4 | 4 +- configure | 21 +++++--- configure.in | 6 ++- depends/libhdfs3/src/client/hdfs.h | 2 + src/Makefile | 20 +++++-- src/include/cwrapper/instrument.h | 8 +++ 9 files changed, 158 insertions(+), 17 deletions(-) diff --git a/.github/scripts/toolchain.sh b/.github/scripts/toolchain.sh new file mode 100644 index 0000000..1e76375 --- /dev/null +++ b/.github/scripts/toolchain.sh @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export HAWQ_TOOLCHAIN_PATH=$(cd "$( dirname "${BASH_SOURCE[0]-$0}" )" && pwd)/download + +# GitHub Release for third-party package +# https://github.com/apache/hawq/releases/tag/thirdparty +REPO=https://github.com/apache/hawq/releases/download/thirdparty/ + +# ISA and OS checking +arch=$(uname -m) +case $arch in + x86_64) ;; + *) echo "Not supported on ISA $arch"; return 1; ;; +esac +os=$(uname -s) +case $os in + Darwin) ;; + Linux) ;; + *) echo "Not supported on OS $os"; return 1; ;; +esac + + +### +### macOS +### +if [[ $(uname -s) == Darwin ]]; then + if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-Darwin ]; then + (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/dependency-Darwin.tar.xz) + (cd $HAWQ_TOOLCHAIN_PATH && tar xJf dependency-Darwin.tar.xz -C $HAWQ_TOOLCHAIN_PATH) + ln -snf dependency-Darwin $HAWQ_TOOLCHAIN_PATH/dependency + + for file in $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/bin -name '*' -type f) $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/lib -name '*.dylib' -type f); do + if [[ $(file $file | grep Mach-O) ]]; then + install_name_tool -add_rpath $HAWQ_TOOLCHAIN_PATH/dependency/package/lib $file 2>/dev/null || true + fi + done + install_name_tool -add_rpath $HAWQ_TOOLCHAIN_PATH/dependency/package/lib/perl5/5.28.0/darwin-thread-multi-2level/CORE/ $HAWQ_TOOLCHAIN_PATH/dependency/package/bin/perl + fi + + export MAKEFLAGS=-j$(sysctl -n hw.ncpu) + export CFLAGS="$CFLAGS -w" + export LDFLAGS="$LDFLAGS -Wl,-rpath,$HAWQ_TOOLCHAIN_PATH/dependency/package/lib" +fi + + + +### +### Linux +### +if [[ $(uname -s) == Linux ]]; then + if [ ! -d $HAWQ_TOOLCHAIN_PATH/gcc ]; then + (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/gcc-7.4.0-x86_64-linux-sles11.4.tar.xz) + (cd $HAWQ_TOOLCHAIN_PATH && tar xJf gcc-7.4.0-x86_64-linux-sles11.4.tar.xz -C $HAWQ_TOOLCHAIN_PATH) + ln -snf gcc-7.4.0-x86_64-linux-sles11.4 $HAWQ_TOOLCHAIN_PATH/gcc + fi + if [ ! -d $HAWQ_TOOLCHAIN_PATH/cmake ]; then + (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/cmake-3.12.4-Linux-x86_64.tar.gz) + (cd $HAWQ_TOOLCHAIN_PATH && tar xzf cmake-3.12.4-Linux-x86_64.tar.gz -C $HAWQ_TOOLCHAIN_PATH) + ln -snf cmake-3.12.4-Linux-x86_64 $HAWQ_TOOLCHAIN_PATH/cmake + fi + if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-gcc-x86_64-Linux/ ]; then + (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/dependency-gcc-x86_64-Linux.tar.gz) + (cd $HAWQ_TOOLCHAIN_PATH && tar xzf dependency-gcc-x86_64-Linux.tar.gz -C $HAWQ_TOOLCHAIN_PATH) + ln -snf dependency-gcc-x86_64-Linux $HAWQ_TOOLCHAIN_PATH/dependency + fi + + export PATH=$HAWQ_TOOLCHAIN_PATH/gcc/bin:$HAWQ_TOOLCHAIN_PATH/cmake/bin:$PATH + export LD_LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:$LD_LIBRARY_PATH + + export CPATH=$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/:$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/x86_64-pc-linux-gnu/ + export CPATH=$CPATH:/usr/include/x86_64-linux-gnu/ + export LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:/usr/lib/x86_64-linux-gnu/ + + unset CPPFLAGS + export CFLAGS='-std=gnu11 -fno-use-linker-plugin' + export CXXFLAGS='-fpermissive -fno-use-linker-plugin' + unset LDFLAGS + + export CC=gcc + export CXX=g++ + export LD=ld + + export MAKEFLAGS=-j$(nproc) +fi + + + +### +find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name CMakeCache.txt -delete +find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name '*build_timestamp' -delete +rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/include/hdfs +rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/lib/libhdfs3* +source $HAWQ_TOOLCHAIN_PATH/dependency/package/env.sh diff --git a/GNUmakefile.in b/GNUmakefile.in index a991c4f..d85fa8c 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -10,6 +10,7 @@ include $(top_builddir)/src/Makefile.global all: # $(MAKE) -C doc $@ + $(MAKE) -C depends/libhdfs3 $@ $(MAKE) -C src $@ $(MAKE) -C config $@ $(MAKE) -C contrib $@ @@ -19,6 +20,7 @@ all: install: # $(MAKE) -C doc $@ + $(MAKE) -C depends/libhdfs3 $@ $(MAKE) -C src $@ $(MAKE) -C config $@ $(MAKE) -C contrib $@ @@ -56,6 +58,7 @@ feature-test-clean: # it's not built by default clean: # $(MAKE) -C doc $@ + $(MAKE) -C depends/libhdfs3 $@ $(MAKE) -C contrib $@ $(MAKE) -C src $@ $(MAKE) -C config $@ @@ -72,6 +75,7 @@ clean: # will be gone too soon. distclean maintainer-clean: # -$(MAKE) -C doc $@ + $(MAKE) -C depends/libhdfs3 $@ -$(MAKE) -C contrib $@ -$(MAKE) -C config $@ -$(MAKE) -C tools $@ diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 88fbbb7..fcd56de 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -73,8 +73,8 @@ int does_int64_work() return 0; return 1; } -main() { - exit(! does_int64_work()); +int main() { + return(! does_int64_work()); }], [Ac_cachevar=yes], [Ac_cachevar=no], diff --git a/config/c-library.m4 b/config/c-library.m4 index e4b2c0b..8a7383e 100644 --- a/config/c-library.m4 +++ b/config/c-library.m4 @@ -250,8 +250,8 @@ int does_int64_snprintf_work() return 0; /* either multiply or snprintf is busted */ return 1; } -main() { - exit(! does_int64_snprintf_work()); +int main() { + return(! does_int64_snprintf_work()); }], [pgac_cv_snprintf_long_long_int_format=$pgac_format; break], [], diff --git a/configure b/configure index ec3303e..7d39b7f 100755 --- a/configure +++ b/configure @@ -6711,7 +6711,7 @@ $as_echo "$as_me: WARNING: *** Include directory $dir does not exist." >&2;} fi done IFS=$ac_save_IFS - +INCLUDES="$INCLUDES -I/usr/local/hawq/include" # @@ -6731,12 +6731,14 @@ $as_echo "$as_me: WARNING: *** Library directory $dir does not exist." >&2;} fi done IFS=$ac_save_IFS - +LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib" # # Assignments # +CFLAGS="$CFLAGS -w" +CXXFLAGS="$CXXFLAGS -w" CPPFLAGS="$CPPFLAGS $INCLUDES" LDFLAGS="$LDFLAGS $LIBDIRS" @@ -15246,8 +15248,8 @@ int does_int64_work() return 0; return 1; } -main() { - exit(! does_int64_work()); +int main() { + return(! does_int64_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -15326,8 +15328,8 @@ int does_int64_work() return 0; return 1; } -main() { - exit(! does_int64_work()); +int main() { + return(! does_int64_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -15419,8 +15421,8 @@ int does_int64_snprintf_work() return 0; /* either multiply or snprintf is busted */ return 1; } -main() { - exit(! does_int64_snprintf_work()); +int main() { + return(! does_int64_snprintf_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -17094,6 +17096,8 @@ fi ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk src/pl/pljava/src/java/Makefile.global" +ac_config_files="$ac_config_files depends/libhdfs3/Makefile.global" + ac_config_files="$ac_config_files src/Makefile.global" ac_config_files="$ac_config_files ranger-plugin/Makefile.global" @@ -17823,6 +17827,7 @@ do case $ac_config_target in "src/backend/port/tas.s") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/tas.s:src/backend/port/tas/${tas_file}" ;; "GNUmakefile") CONFIG_FILES="$CONFIG_FILES GNUmakefile" ;; + "depends/libhdfs3/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libhdfs3/Makefile.global" ;; "src/VERSIONS.mk") CONFIG_FILES="$CONFIG_FILES src/VERSIONS.mk" ;; "src/pl/pljava/src/java/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/pl/pljava/src/java/Makefile.global" ;; "src/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/Makefile.global" ;; diff --git a/configure.in b/configure.in index f258699..25cd5a2 100644 --- a/configure.in +++ b/configure.in @@ -924,6 +924,7 @@ for dir in $with_includes $SRCH_INC; do fi done IFS=$ac_save_IFS +INCLUDES="$INCLUDES -I/usr/local/hawq/include" AC_SUBST(INCLUDES) @@ -943,12 +944,14 @@ for dir in $LIBRARY_DIRS $SRCH_LIB; do fi done IFS=$ac_save_IFS - +LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib" # # Assignments # +CFLAGS="$CFLAGS -w" +CXXFLAGS="$CXXFLAGS -w" CPPFLAGS="$CPPFLAGS $INCLUDES" LDFLAGS="$LDFLAGS $LIBDIRS" @@ -2205,6 +2208,7 @@ fi AC_SUBST(vpath_build) AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk src/pl/pljava/src/java/Makefile.global]) +AC_CONFIG_FILES([depends/libhdfs3/Makefile.global]) AC_CONFIG_FILES([src/Makefile.global]) AC_CONFIG_FILES([ranger-plugin/Makefile.global]) diff --git a/depends/libhdfs3/src/client/hdfs.h b/depends/libhdfs3/src/client/hdfs.h index f8b61ea..a60c150 100644 --- a/depends/libhdfs3/src/client/hdfs.h +++ b/depends/libhdfs3/src/client/hdfs.h @@ -726,6 +726,8 @@ Namenode * hdfsGetHANamenodesWithConfig(const char * conf, const char * nameserv void hdfsFreeNamenodeInformation(Namenode * namenodes, int size); typedef struct BlockLocation { + uint32_t rangeId; + uint32_t replicaGroupId; int corrupt; // If the block is corrupt int numOfNodes; // Number of Datanodes which keep the block char ** hosts; // Datanode hostnames diff --git a/src/Makefile b/src/Makefile index b707cb3..e461abd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -13,7 +13,7 @@ top_builddir = .. include Makefile.global -all install installdirs uninstall distprep: install-cwrapper +all install installdirs uninstall distprep: check-and-install-cwrapper $(MAKE) -C port $@ $(MAKE) -C timezone $@ $(MAKE) -C backend $@ @@ -26,7 +26,8 @@ all install installdirs uninstall distprep: install-cwrapper $(MAKE) -C test/regress $@ $(MAKE) -C tools/fsync $@ -install-cwrapper: +check-and-install-cwrapper: +ifeq ("$(wildcard $(DEPENDENCY_PATH)/include/cwrapper-installed)", "") rm -rf $(DEPENDENCY_PATH)/include/executor rm -rf $(DEPENDENCY_PATH)/include/scheduler rm -rf $(DEPENDENCY_PATH)/include/magma @@ -54,6 +55,8 @@ install-cwrapper: $(INSTALL_DATA) include/cwrapper/orc-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/' $(INSTALL_DATA) include/cwrapper/text-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/' $(INSTALL_DATA) include/cwrapper/magma-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/' + touch $(DEPENDENCY_PATH)/include/cwrapper-installed +endif feature-test: $(MAKE) -C test feature-test @@ -86,7 +89,16 @@ uninstall: uninstall-local uninstall-local: rm -f $(addprefix '$(DESTDIR)$(pgxsdir)/$(subdir)'/, Makefile.global Makefile.port Makefile.shlib nls-global.mk) -clean: +clear-cwrapper: + rm -rf $(DEPENDENCY_PATH)/include/executor + rm -rf $(DEPENDENCY_PATH)/include/scheduler + rm -rf $(DEPENDENCY_PATH)/include/magma + rm -rf $(DEPENDENCY_PATH)/include/dbcommon + rm -rf $(DEPENDENCY_PATH)/include/univplan + rm -rf $(DEPENDENCY_PATH)/include/storage + rm -f $(DEPENDENCY_PATH)/include/cwrapper-installed + +clean: clear-cwrapper $(MAKE) -C port $@ $(MAKE) -C timezone $@ $(MAKE) -C backend $@ @@ -102,7 +114,7 @@ clean: $(MAKE) -C test/feature $@ $(MAKE) -C tools/fsync $@ -distclean maintainer-clean: +distclean maintainer-clean: clear-cwrapper $(MAKE) -C port $@ $(MAKE) -C timezone $@ $(MAKE) -C backend $@ diff --git a/src/include/cwrapper/instrument.h b/src/include/cwrapper/instrument.h index e5c3d11..7da4a08 100644 --- a/src/include/cwrapper/instrument.h +++ b/src/include/cwrapper/instrument.h @@ -10,6 +10,10 @@ #include <stdint.h> +#ifdef __cplusplus +extern "C" { +#endif + typedef struct MyInstrumentation { struct MyInstrumentation *leftTree; struct MyInstrumentation *rightTree; @@ -51,4 +55,8 @@ __attribute__((weak)) void MyInstrStopNode(MyInstrumentation *instr, uint64_t nT __attribute__((weak)) void MyInstrEndLoop(MyInstrumentation *instr) {} +#ifdef __cplusplus +} +#endif + #endif // DBCOMMON_SRC_DBCOMMON_UTILS_INSTRUMENT_H_