This is an automated email from the ASF dual-hosted git repository. wangchunling pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hawq.git
The following commit(s) were added to refs/heads/master by this push: new 4cbd805 HAWQ-1780. Add GitHub Action Step to Test against Running Instance 4cbd805 is described below commit 4cbd8057e5f27802c3e3a709bb0f735c71315c6b Author: Chiyang Wan <chiyang10...@gmail.com> AuthorDate: Thu Jan 21 10:27:06 2021 +0800 HAWQ-1780. Add GitHub Action Step to Test against Running Instance It fixes the install_name of libhdfs and libyarn with @rpath prefix, which is required when loading executable in different path, and enforces searching headers and libraries in those self-contained first. --- .github/workflows/build.yml | 34 +++++++++++++++--- .github/workflows/scripts/init_hawq.sh | 64 +++++++++++++++++++++++++++++++++ .github/workflows/scripts/init_hdfs.sh | 52 +++++++++++++++++++++++++++ .github/workflows/scripts/init_macos.sh | 49 +++++++++++++++++++++++++ depends/libhdfs3/CMakeLists.txt | 1 + depends/libyarn/CMakeLists.txt | 1 + src/Makefile.global.in | 8 ++--- 7 files changed, 201 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee4e22f..503109c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ name: Apache HAWQ -on: [push] +on: [push, pull_request] jobs: build-on-macOS: @@ -44,6 +44,7 @@ jobs: install_name_tool -add_rpath $GITHUB_WORKSPACE/dependency-Darwin/package/lib/perl5/5.28.0/darwin-thread-multi-2level/CORE/ $GITHUB_WORKSPACE/dependency-Darwin/package/bin/perl - name: configure + timeout-minutes: 10 run: | source $GITHUB_WORKSPACE/dependency-Darwin/package/env.sh export CFLAGS="$CFLAGS -w" @@ -55,16 +56,41 @@ jobs: - name: build hawq run: | source $GITHUB_WORKSPACE/dependency-Darwin/package/env.sh - make -j$(nproc) - make -j$(nproc) install + make -j$(sysctl -n hw.ncpu) + make -j$(sysctl -n hw.ncpu) install - name: build feature-test run: | source $GITHUB_WORKSPACE/dependency-Darwin/package/env.sh - make -j$(nproc) feature-test + make -j$(sysctl -n hw.ncpu) feature-test - name: test executable run: | + for file in $(find /tmp/hawq/bin -name '*' -type f); do + if [[ $(file $file | grep Mach-O) ]]; then + install_name_tool -add_rpath /tmp/hawq/lib $file; + fi + done source /tmp/hawq/greenplum_path.sh postgres -V src/test/feature/feature-test --gtest_list_tests + + - name: install HDFS + run: | + export HOMEBREW_NO_INSTALL_CLEANUP=1 + brew install hadoop + + - name: initilize macOS + run: .github/workflows/scripts/init_macos.sh + + - name: initilize HDFS + run: | + export HADOOP_HOME=/usr/local/opt/hadoop/libexec + .github/workflows/scripts/init_hdfs.sh + + - name: initilize HAWQ + run: | + source /tmp/hawq/greenplum_path.sh + .github/workflows/scripts/init_hawq.sh + psql -d postgres -c 'create database hawq_feature_test_db;' + src/test/feature/feature-test --gtest_filter=TestDatabase.BasicTest diff --git a/.github/workflows/scripts/init_hawq.sh b/.github/workflows/scripts/init_hawq.sh new file mode 100755 index 0000000..0793ebd --- /dev/null +++ b/.github/workflows/scripts/init_hawq.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + + + +# Configure +tee $GPHOME/etc/hawq-site.xml << EOF_hawq_site +<configuration> + <property> + <name>hawq_dfs_url</name> + <value>localhost:8020/hawq_default</value> + <description>URL for accessing HDFS.</description> + </property> + <property> + <name>hawq_master_address_host</name> + <value>localhost</value> + </property> + <property> + <name>hawq_master_address_port</name> + <value>5432</value> + </property> + <property> + <name>hawq_segment_address_port</name> + <value>40000</value> + </property> + <property> + <name>hawq_master_directory</name> + <value>/tmp/db_data/hawq-data-directory/masterdd</value> + </property> + <property> + <name>hawq_segment_directory</name> + <value>/tmp/db_data/hawq-data-directory/segmentdd</value> + </property> + <property> + <name>hawq_master_temp_directory</name> + <value>/tmp</value> + </property> + <property> + <name>hawq_segment_temp_directory</name> + <value>/tmp</value> + </property> +</configuration> +EOF_hawq_site + +# Initialize +rm -rf /opt/dependency* +rm -rf /tmp/db_data/hawq-data-directory +install -d /tmp/db_data/hawq-data-directory/masterdd +install -d /tmp/db_data/hawq-data-directory/segmentdd +hawq init cluster -a diff --git a/.github/workflows/scripts/init_hdfs.sh b/.github/workflows/scripts/init_hdfs.sh new file mode 100755 index 0000000..0ab2094 --- /dev/null +++ b/.github/workflows/scripts/init_hdfs.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + + + +# Configure +tee $HADOOP_HOME/etc/hadoop/core-site.xml << EOF_core_site +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://localhost:8020</value> + </property> +</configuration> +EOF_core_site +tee $HADOOP_HOME/etc/hadoop/hdfs-site.xml << EOF_hdfs_site +<configuration> + <property> + <name>dfs.namenode.name.dir</name> + <value>file:///tmp/db_data/hdfs/name</value> + </property> + <property> + <name>dfs.datanode.data.dir</name> + <value>file:///tmp/db_data/hdfs/data</value> + </property> +</configuration> +EOF_hdfs_site + +# Initialize +install -d /tmp/db_data/hdfs/name +install -d /tmp/db_data/hdfs/data +hdfs namenode -format + +# Start +$HADOOP_HOME/sbin/start-dfs.sh + +# Connect +hdfs dfsadmin -report +hdfs dfs -ls / diff --git a/.github/workflows/scripts/init_macos.sh b/.github/workflows/scripts/init_macos.sh new file mode 100755 index 0000000..f02aa21 --- /dev/null +++ b/.github/workflows/scripts/init_macos.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + + + +# Setup passphraseless ssh +sudo systemsetup -setremotelogin on +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 0700 ~/.ssh +chmod 0600 ~/.ssh/authorized_keys + +tee -a ~/.ssh/config <<EOF_ssh_config +Host * + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null +EOF_ssh_config + +ssh -v localhost whoami + +# Configure system kernel state +sudo tee /etc/sysctl.conf << EOF_sysctl +kern.sysv.shmmax=2147483648 +kern.sysv.shmmin=1 +kern.sysv.shmmni=64 +kern.sysv.shmseg=16 +kern.sysv.shmall=524288 +kern.maxfiles=65535 +kern.maxfilesperproc=65536 +kern.corefile=/cores/core.%N.%P +EOF_sysctl +</etc/sysctl.conf xargs sudo sysctl + +# Add data folder +sudo install -o $USER -d /tmp/db_data/ diff --git a/depends/libhdfs3/CMakeLists.txt b/depends/libhdfs3/CMakeLists.txt index 4694942..ba158ed 100644 --- a/depends/libhdfs3/CMakeLists.txt +++ b/depends/libhdfs3/CMakeLists.txt @@ -3,6 +3,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(libhdfs3) SET(CMAKE_VERBOSE_MAKEFILE ON CACHE STRING "Verbose build." FORCE) +SET(CMAKE_MACOSX_RPATH 1) IF(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) MESSAGE(FATAL_ERROR "cannot build the project in the source directory! Out-of-source build is enforced!") diff --git a/depends/libyarn/CMakeLists.txt b/depends/libyarn/CMakeLists.txt index e7719d4..7ad2a81 100644 --- a/depends/libyarn/CMakeLists.txt +++ b/depends/libyarn/CMakeLists.txt @@ -4,6 +4,7 @@ PROJECT(libyarn) SET(CMAKE_VERBOSE_MAKEFILE ON CACHE STRING "Verbose build." FORCE) SET(CMAKE_FIND_ROOT_PATH "/Users/weikui/Documents/project/osx106_x86") +SET(CMAKE_MACOSX_RPATH 1) IF(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) MESSAGE(FATAL_ERROR "cannot build the project in the source directory! Out-of-source build is enforced!") diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 51dffad..2b47748 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -221,8 +221,6 @@ COLLATEINDEX = @COLLATEINDEX@ CPP = @CPP@ CPPFLAGS = @CPPFLAGS@ -CPPFLAGS += -I$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/include -CPPFLAGS += -I$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/include CPPFLAGS += -I/usr/local/hawq/include ifdef PGXS @@ -237,7 +235,9 @@ endif # not PGXS CC = @CC@ GCC = @GCC@ SUN_STUDIO_CC = @SUN_STUDIO_CC@ -CFLAGS = @CFLAGS@ +CFLAGS = -I$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/include +CFLAGS += -I$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/include +CFLAGS += @CFLAGS@ CFLAGS_SSE42 = @CFLAGS_SSE42@ # Kind-of compilers @@ -272,9 +272,9 @@ ifdef PGXS else LDFLAGS = -L$(top_builddir)/src/port endif -LDFLAGS += @LDFLAGS@ LDFLAGS += -L$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/lib LDFLAGS += -L$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/lib +LDFLAGS += @LDFLAGS@ LDFLAGS_EX = @LDFLAGS_EX@ # LDFLAGS_SL might have already been assigned by calling makefile