This is an automated email from the ASF dual-hosted git repository.
ztao1987 pushed a commit to branch taoz
in repository https://gitbox.apache.org/repos/asf/hawq.git
The following commit(s) were added to refs/heads/taoz by this push:
new 3922ae7 HAWQ-1799. include libhdfs3 in building path
3922ae7 is described below
commit 3922ae775d87a928a6eefb3adecfa9c8d1c198f0
Author: ztao1987 <[email protected]>
AuthorDate: Tue Aug 3 10:13:07 2021 +0800
HAWQ-1799. include libhdfs3 in building path
---
.github/scripts/toolchain.sh | 106 +++++++++++++++++++++++++++++++++++++
GNUmakefile.in | 4 ++
config/c-compiler.m4 | 4 +-
config/c-library.m4 | 4 +-
configure | 21 +++++---
configure.in | 6 ++-
depends/libhdfs3/src/client/hdfs.h | 2 +
src/Makefile | 20 +++++--
src/include/cwrapper/instrument.h | 8 +++
9 files changed, 158 insertions(+), 17 deletions(-)
diff --git a/.github/scripts/toolchain.sh b/.github/scripts/toolchain.sh
new file mode 100644
index 0000000..1e76375
--- /dev/null
+++ b/.github/scripts/toolchain.sh
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_TOOLCHAIN_PATH=$(cd "$( dirname "${BASH_SOURCE[0]-$0}" )" &&
pwd)/download
+
+# GitHub Release for third-party package
+# https://github.com/apache/hawq/releases/tag/thirdparty
+REPO=https://github.com/apache/hawq/releases/download/thirdparty/
+
+# ISA and OS checking
+arch=$(uname -m)
+case $arch in
+ x86_64) ;;
+ *) echo "Not supported on ISA $arch"; return 1; ;;
+esac
+os=$(uname -s)
+case $os in
+ Darwin) ;;
+ Linux) ;;
+ *) echo "Not supported on OS $os"; return 1; ;;
+esac
+
+
+###
+### macOS
+###
+if [[ $(uname -s) == Darwin ]]; then
+ if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-Darwin ]; then
+ (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/dependency-Darwin.tar.xz)
+ (cd $HAWQ_TOOLCHAIN_PATH && tar xJf dependency-Darwin.tar.xz -C
$HAWQ_TOOLCHAIN_PATH)
+ ln -snf dependency-Darwin $HAWQ_TOOLCHAIN_PATH/dependency
+
+ for file in $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/bin -name '*'
-type f) $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/lib -name '*.dylib'
-type f); do
+ if [[ $(file $file | grep Mach-O) ]]; then
+ install_name_tool -add_rpath
$HAWQ_TOOLCHAIN_PATH/dependency/package/lib $file 2>/dev/null || true
+ fi
+ done
+ install_name_tool -add_rpath
$HAWQ_TOOLCHAIN_PATH/dependency/package/lib/perl5/5.28.0/darwin-thread-multi-2level/CORE/
$HAWQ_TOOLCHAIN_PATH/dependency/package/bin/perl
+ fi
+
+ export MAKEFLAGS=-j$(sysctl -n hw.ncpu)
+ export CFLAGS="$CFLAGS -w"
+ export LDFLAGS="$LDFLAGS
-Wl,-rpath,$HAWQ_TOOLCHAIN_PATH/dependency/package/lib"
+fi
+
+
+
+###
+### Linux
+###
+if [[ $(uname -s) == Linux ]]; then
+ if [ ! -d $HAWQ_TOOLCHAIN_PATH/gcc ]; then
+ (cd $HAWQ_TOOLCHAIN_PATH && curl -OL
$REPO/gcc-7.4.0-x86_64-linux-sles11.4.tar.xz)
+ (cd $HAWQ_TOOLCHAIN_PATH && tar xJf gcc-7.4.0-x86_64-linux-sles11.4.tar.xz
-C $HAWQ_TOOLCHAIN_PATH)
+ ln -snf gcc-7.4.0-x86_64-linux-sles11.4 $HAWQ_TOOLCHAIN_PATH/gcc
+ fi
+ if [ ! -d $HAWQ_TOOLCHAIN_PATH/cmake ]; then
+ (cd $HAWQ_TOOLCHAIN_PATH && curl -OL
$REPO/cmake-3.12.4-Linux-x86_64.tar.gz)
+ (cd $HAWQ_TOOLCHAIN_PATH && tar xzf cmake-3.12.4-Linux-x86_64.tar.gz -C
$HAWQ_TOOLCHAIN_PATH)
+ ln -snf cmake-3.12.4-Linux-x86_64 $HAWQ_TOOLCHAIN_PATH/cmake
+ fi
+ if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-gcc-x86_64-Linux/ ]; then
+ (cd $HAWQ_TOOLCHAIN_PATH && curl -OL
$REPO/dependency-gcc-x86_64-Linux.tar.gz)
+ (cd $HAWQ_TOOLCHAIN_PATH && tar xzf dependency-gcc-x86_64-Linux.tar.gz -C
$HAWQ_TOOLCHAIN_PATH)
+ ln -snf dependency-gcc-x86_64-Linux $HAWQ_TOOLCHAIN_PATH/dependency
+ fi
+
+ export PATH=$HAWQ_TOOLCHAIN_PATH/gcc/bin:$HAWQ_TOOLCHAIN_PATH/cmake/bin:$PATH
+ export LD_LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:$LD_LIBRARY_PATH
+
+ export
CPATH=$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/:$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/x86_64-pc-linux-gnu/
+ export CPATH=$CPATH:/usr/include/x86_64-linux-gnu/
+ export
LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:/usr/lib/x86_64-linux-gnu/
+
+ unset CPPFLAGS
+ export CFLAGS='-std=gnu11 -fno-use-linker-plugin'
+ export CXXFLAGS='-fpermissive -fno-use-linker-plugin'
+ unset LDFLAGS
+
+ export CC=gcc
+ export CXX=g++
+ export LD=ld
+
+ export MAKEFLAGS=-j$(nproc)
+fi
+
+
+
+###
+find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name CMakeCache.txt -delete
+find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name '*build_timestamp' -delete
+rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/include/hdfs
+rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/lib/libhdfs3*
+source $HAWQ_TOOLCHAIN_PATH/dependency/package/env.sh
diff --git a/GNUmakefile.in b/GNUmakefile.in
index a991c4f..d85fa8c 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -10,6 +10,7 @@ include $(top_builddir)/src/Makefile.global
all:
# $(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib $@
@@ -19,6 +20,7 @@ all:
install:
# $(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib $@
@@ -56,6 +58,7 @@ feature-test-clean:
# it's not built by default
clean:
# $(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
$(MAKE) -C contrib $@
$(MAKE) -C src $@
$(MAKE) -C config $@
@@ -72,6 +75,7 @@ clean:
# will be gone too soon.
distclean maintainer-clean:
# -$(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
-$(MAKE) -C contrib $@
-$(MAKE) -C config $@
-$(MAKE) -C tools $@
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 88fbbb7..fcd56de 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -73,8 +73,8 @@ int does_int64_work()
return 0;
return 1;
}
-main() {
- exit(! does_int64_work());
+int main() {
+ return(! does_int64_work());
}],
[Ac_cachevar=yes],
[Ac_cachevar=no],
diff --git a/config/c-library.m4 b/config/c-library.m4
index e4b2c0b..8a7383e 100644
--- a/config/c-library.m4
+++ b/config/c-library.m4
@@ -250,8 +250,8 @@ int does_int64_snprintf_work()
return 0; /* either multiply or snprintf is busted */
return 1;
}
-main() {
- exit(! does_int64_snprintf_work());
+int main() {
+ return(! does_int64_snprintf_work());
}],
[pgac_cv_snprintf_long_long_int_format=$pgac_format; break],
[],
diff --git a/configure b/configure
index ec3303e..7d39b7f 100755
--- a/configure
+++ b/configure
@@ -6711,7 +6711,7 @@ $as_echo "$as_me: WARNING: *** Include directory $dir
does not exist." >&2;}
fi
done
IFS=$ac_save_IFS
-
+INCLUDES="$INCLUDES -I/usr/local/hawq/include"
#
@@ -6731,12 +6731,14 @@ $as_echo "$as_me: WARNING: *** Library directory $dir
does not exist." >&2;}
fi
done
IFS=$ac_save_IFS
-
+LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib"
#
# Assignments
#
+CFLAGS="$CFLAGS -w"
+CXXFLAGS="$CXXFLAGS -w"
CPPFLAGS="$CPPFLAGS $INCLUDES"
LDFLAGS="$LDFLAGS $LIBDIRS"
@@ -15246,8 +15248,8 @@ int does_int64_work()
return 0;
return 1;
}
-main() {
- exit(! does_int64_work());
+int main() {
+ return(! does_int64_work());
}
_ACEOF
if ac_fn_c_try_run "$LINENO"; then :
@@ -15326,8 +15328,8 @@ int does_int64_work()
return 0;
return 1;
}
-main() {
- exit(! does_int64_work());
+int main() {
+ return(! does_int64_work());
}
_ACEOF
if ac_fn_c_try_run "$LINENO"; then :
@@ -15419,8 +15421,8 @@ int does_int64_snprintf_work()
return 0; /* either multiply or snprintf is busted */
return 1;
}
-main() {
- exit(! does_int64_snprintf_work());
+int main() {
+ return(! does_int64_snprintf_work());
}
_ACEOF
if ac_fn_c_try_run "$LINENO"; then :
@@ -17094,6 +17096,8 @@ fi
ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk
src/pl/pljava/src/java/Makefile.global"
+ac_config_files="$ac_config_files depends/libhdfs3/Makefile.global"
+
ac_config_files="$ac_config_files src/Makefile.global"
ac_config_files="$ac_config_files ranger-plugin/Makefile.global"
@@ -17823,6 +17827,7 @@ do
case $ac_config_target in
"src/backend/port/tas.s") CONFIG_LINKS="$CONFIG_LINKS
src/backend/port/tas.s:src/backend/port/tas/${tas_file}" ;;
"GNUmakefile") CONFIG_FILES="$CONFIG_FILES GNUmakefile" ;;
+ "depends/libhdfs3/Makefile.global") CONFIG_FILES="$CONFIG_FILES
depends/libhdfs3/Makefile.global" ;;
"src/VERSIONS.mk") CONFIG_FILES="$CONFIG_FILES src/VERSIONS.mk" ;;
"src/pl/pljava/src/java/Makefile.global") CONFIG_FILES="$CONFIG_FILES
src/pl/pljava/src/java/Makefile.global" ;;
"src/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/Makefile.global" ;;
diff --git a/configure.in b/configure.in
index f258699..25cd5a2 100644
--- a/configure.in
+++ b/configure.in
@@ -924,6 +924,7 @@ for dir in $with_includes $SRCH_INC; do
fi
done
IFS=$ac_save_IFS
+INCLUDES="$INCLUDES -I/usr/local/hawq/include"
AC_SUBST(INCLUDES)
@@ -943,12 +944,14 @@ for dir in $LIBRARY_DIRS $SRCH_LIB; do
fi
done
IFS=$ac_save_IFS
-
+LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib"
#
# Assignments
#
+CFLAGS="$CFLAGS -w"
+CXXFLAGS="$CXXFLAGS -w"
CPPFLAGS="$CPPFLAGS $INCLUDES"
LDFLAGS="$LDFLAGS $LIBDIRS"
@@ -2205,6 +2208,7 @@ fi
AC_SUBST(vpath_build)
AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk
src/pl/pljava/src/java/Makefile.global])
+AC_CONFIG_FILES([depends/libhdfs3/Makefile.global])
AC_CONFIG_FILES([src/Makefile.global])
AC_CONFIG_FILES([ranger-plugin/Makefile.global])
diff --git a/depends/libhdfs3/src/client/hdfs.h
b/depends/libhdfs3/src/client/hdfs.h
index f8b61ea..a60c150 100644
--- a/depends/libhdfs3/src/client/hdfs.h
+++ b/depends/libhdfs3/src/client/hdfs.h
@@ -726,6 +726,8 @@ Namenode * hdfsGetHANamenodesWithConfig(const char * conf,
const char * nameserv
void hdfsFreeNamenodeInformation(Namenode * namenodes, int size);
typedef struct BlockLocation {
+ uint32_t rangeId;
+ uint32_t replicaGroupId;
int corrupt; // If the block is corrupt
int numOfNodes; // Number of Datanodes which keep the block
char ** hosts; // Datanode hostnames
diff --git a/src/Makefile b/src/Makefile
index b707cb3..e461abd 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -13,7 +13,7 @@ top_builddir = ..
include Makefile.global
-all install installdirs uninstall distprep: install-cwrapper
+all install installdirs uninstall distprep: check-and-install-cwrapper
$(MAKE) -C port $@
$(MAKE) -C timezone $@
$(MAKE) -C backend $@
@@ -26,7 +26,8 @@ all install installdirs uninstall distprep: install-cwrapper
$(MAKE) -C test/regress $@
$(MAKE) -C tools/fsync $@
-install-cwrapper:
+check-and-install-cwrapper:
+ifeq ("$(wildcard $(DEPENDENCY_PATH)/include/cwrapper-installed)", "")
rm -rf $(DEPENDENCY_PATH)/include/executor
rm -rf $(DEPENDENCY_PATH)/include/scheduler
rm -rf $(DEPENDENCY_PATH)/include/magma
@@ -54,6 +55,8 @@ install-cwrapper:
$(INSTALL_DATA) include/cwrapper/orc-format-c.h
'$(DEPENDENCY_PATH)/include/storage/cwrapper/'
$(INSTALL_DATA) include/cwrapper/text-format-c.h
'$(DEPENDENCY_PATH)/include/storage/cwrapper/'
$(INSTALL_DATA) include/cwrapper/magma-format-c.h
'$(DEPENDENCY_PATH)/include/storage/cwrapper/'
+ touch $(DEPENDENCY_PATH)/include/cwrapper-installed
+endif
feature-test:
$(MAKE) -C test feature-test
@@ -86,7 +89,16 @@ uninstall: uninstall-local
uninstall-local:
rm -f $(addprefix '$(DESTDIR)$(pgxsdir)/$(subdir)'/, Makefile.global
Makefile.port Makefile.shlib nls-global.mk)
-clean:
+clear-cwrapper:
+ rm -rf $(DEPENDENCY_PATH)/include/executor
+ rm -rf $(DEPENDENCY_PATH)/include/scheduler
+ rm -rf $(DEPENDENCY_PATH)/include/magma
+ rm -rf $(DEPENDENCY_PATH)/include/dbcommon
+ rm -rf $(DEPENDENCY_PATH)/include/univplan
+ rm -rf $(DEPENDENCY_PATH)/include/storage
+ rm -f $(DEPENDENCY_PATH)/include/cwrapper-installed
+
+clean: clear-cwrapper
$(MAKE) -C port $@
$(MAKE) -C timezone $@
$(MAKE) -C backend $@
@@ -102,7 +114,7 @@ clean:
$(MAKE) -C test/feature $@
$(MAKE) -C tools/fsync $@
-distclean maintainer-clean:
+distclean maintainer-clean: clear-cwrapper
$(MAKE) -C port $@
$(MAKE) -C timezone $@
$(MAKE) -C backend $@
diff --git a/src/include/cwrapper/instrument.h
b/src/include/cwrapper/instrument.h
index e5c3d11..7da4a08 100644
--- a/src/include/cwrapper/instrument.h
+++ b/src/include/cwrapper/instrument.h
@@ -10,6 +10,10 @@
#include <stdint.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
typedef struct MyInstrumentation {
struct MyInstrumentation *leftTree;
struct MyInstrumentation *rightTree;
@@ -51,4 +55,8 @@ __attribute__((weak)) void MyInstrStopNode(MyInstrumentation
*instr, uint64_t nT
__attribute__((weak)) void MyInstrEndLoop(MyInstrumentation *instr) {}
+#ifdef __cplusplus
+}
+#endif
+
#endif // DBCOMMON_SRC_DBCOMMON_UTILS_INSTRUMENT_H_
