This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch apple-m1 in repository https://gitbox.apache.org/repos/asf/kudu.git
commit b0c8d2f59d69a005e69fb779817b6d80aa9b595e Author: Grant Henke <[email protected]> AuthorDate: Thu Nov 26 23:38:51 2020 -0600 WIP: Build on Apple M1 (arm64) This is a WIP Patch to build Kudu on Apple M1 and Big Sur (OSX 11). Environment setup log: # Setup passwordless sudo sudo visudo # Edit the line "%admin ALL=(ALL) ALL" to "%admin ALL=(ALL) NOPASSWD: ALL" # Install xcode sudo xcodebuild -license sudo xcode-select --install # TODO: Document to use xcode git (remove git from below list) # Validate git is comming from xcode # Example: git version 2.24.3 (Apple Git-128) git --version # Install homebrew cd ~ mkdir homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C homebrew sudo mv homebrew /opt/homebrew cd /opt/homebrew/bin ./brew update ./brew doctor # Fix any issues cd ~ # Install Java # Currently Azul has Arm 64-bit macOS releases: # https://www.azul.com/downloads/zulu-community/?os=macos&architecture=arm-64-bit&package=jdk curl https://cdn.azul.com/zulu/bin/zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip -o zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip unzip zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip sudo mv zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64/zulu-8.jdk /Library/Java/JavaVirtualMachines/zulu-8.jdk JAVA_HOME=`sudo /usr/libexec/java_home -v 8` java -version # I recomend installing https://github.com/ohmyzsh/ohmyzsh # Update your ~/.zshrc to include the following: # Add the Homebrew bin to the PATH export PATH="/opt/homebrew/bin:$PATH" # Add the Homebrew OpenSSL to the environment export PATH="/opt/homebrew/opt/openssl/bin:$PATH" export LDFLAGS="-L/opt/homebrew/opt/openssl/lib" export CPPFLAGS="-I/opt/homebrew/opt/openssl/include" export PKG_CONFIG_PATH="/opt/homebrew/opt/openssl/lib/pkgconfig" # Set JAVA_HOME export JAVA_HOME=`sudo /usr/libexec/java_home -v 8` # Reload your .zshrc source ~/.zshrc # TODO: Document the -s # Install the required Kudu packages. brew install -s autoconf automake cmake krb5 libtool openssl pkg-config pstree # ccache doesn't work :( brew install -s ccache Build Log: # Clone and build Kudu mkdir ~/source cd ~/source git clone https://github.com/apache/kudu cd kudu thirdparty/build-if-necessary.sh # TODO: Figure out why cmake cant find g++ # Added -DCMAKE_CXX_COMPILER=/usr/bin/g++ for now. mkdir -p build/debug cd build/debug ../../thirdparty/installed/common/bin/cmake \ -DCMAKE_BUILD_TYPE=debug \ -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl \ -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ ../..make - ctest --- CMakeLists.txt | 34 +++++++++++++++++---------- java/gradle/protobuf.gradle | 5 +++- src/kudu/gutil/CMakeLists.txt | 2 +- src/kudu/gutil/atomicops-internals-macosx.h | 4 +++- thirdparty/download-thirdparty.sh | 2 +- thirdparty/patches/gperftools-osx-arm64.patch | 21 +++++++++++++++++ thirdparty/vars.sh | 2 +- 7 files changed, 52 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3350953..f88f4ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,13 @@ include(CompilerInfo) # compiler flags that are common across debug/release builds execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH_NAME) +message(STATUS "Found ARCH_NAME: ${ARCH_NAME}") +# The Apple backend for aarch64 is called arm64. +# Rename to aarch64 to simplify all of our build checks. +if("${ARCH_NAME}" MATCHES "arm64") + set(ARCH_NAME "aarch64") +endif() +message(STATUS "Using ARCH_NAME: ${ARCH_NAME}") if("${ARCH_NAME}" MATCHES "aarch64") # Certain platforms such as ARM do not use signed chars by default # which causes issues with certain bounds checks. @@ -1084,19 +1091,20 @@ endif() ## Google PerfTools ## ## Disabled with TSAN/ASAN. -if (NOT "${KUDU_USE_ASAN}" AND - NOT "${KUDU_USE_TSAN}") - find_package(GPerf REQUIRED) - ADD_THIRDPARTY_LIB(tcmalloc - STATIC_LIB "${TCMALLOC_STATIC_LIB}" - SHARED_LIB "${TCMALLOC_SHARED_LIB}") - ADD_THIRDPARTY_LIB(profiler - STATIC_LIB "${PROFILER_STATIC_LIB}" - SHARED_LIB "${PROFILER_SHARED_LIB}") - list(APPEND KUDU_BASE_LIBS tcmalloc profiler) - add_definitions("-DTCMALLOC_ENABLED") - set(KUDU_TCMALLOC_AVAILABLE 1) -endif() +# TODO: Figure out why tcmalloc hangs protoc on OSX aarch64 +#if (NOT "${KUDU_USE_ASAN}" AND +# NOT "${KUDU_USE_TSAN}") +# find_package(GPerf REQUIRED) +# ADD_THIRDPARTY_LIB(tcmalloc +# STATIC_LIB "${TCMALLOC_STATIC_LIB}" +# SHARED_LIB "${TCMALLOC_SHARED_LIB}") +# ADD_THIRDPARTY_LIB(profiler +# STATIC_LIB "${PROFILER_STATIC_LIB}" +# SHARED_LIB "${PROFILER_SHARED_LIB}") +# list(APPEND KUDU_BASE_LIBS tcmalloc profiler) +# add_definitions("-DTCMALLOC_ENABLED") +# set(KUDU_TCMALLOC_AVAILABLE 1) +#endif() ## curl find_package(CURL REQUIRED) diff --git a/java/gradle/protobuf.gradle b/java/gradle/protobuf.gradle index 5bcabd3..0ee4c45 100644 --- a/java/gradle/protobuf.gradle +++ b/java/gradle/protobuf.gradle @@ -22,7 +22,10 @@ apply plugin: "com.google.protobuf" // Use the maven protoc artifact instead of local. protobuf { protoc { - artifact = libs.protoc + // TODO: switch between artifacy and thirdparty protoc via command line property + // On Apple aarch64 use thirdparty by default. + path = "$rootDir/../thirdparty/installed/uninstrumented/bin/protoc" + // artifact = libs.protoc } } diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt index 02808d9..460c975 100644 --- a/src/kudu/gutil/CMakeLists.txt +++ b/src/kudu/gutil/CMakeLists.txt @@ -50,7 +50,7 @@ set(GUTIL_SRCS utf/rune.c walltime.cc) -if (NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") +if (NOT "${ARCH_NAME}" MATCHES "aarch64") set(GUTIL_SRCS atomicops-internals-x86.cc ${GUTIL_SRCS}) diff --git a/src/kudu/gutil/atomicops-internals-macosx.h b/src/kudu/gutil/atomicops-internals-macosx.h index a260d3b..c170957 100644 --- a/src/kudu/gutil/atomicops-internals-macosx.h +++ b/src/kudu/gutil/atomicops-internals-macosx.h @@ -274,7 +274,9 @@ inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { // are in a spinlock wait loop and should allow other hyperthreads // to run, not speculate memory access, etc. inline void PauseCPU() { - __asm__ __volatile__("pause" : : : "memory"); + // TODO: Only on OSX aarch64 + __asm__ __volatile__("yield" : : : "memory"); + // __asm__ __volatile__("pause" : : : "memory"); } inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 2edb494..630aa7a 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -199,7 +199,7 @@ fetch_and_patch \ $GPERFTOOLS_SOURCE \ $GPERFTOOLS_PATCHLEVEL \ "patch -p1 < $TP_DIR/patches/gperftools-Replace-namespace-base-with-namespace-tcmalloc.patch" \ - "patch -p1 < $TP_DIR/patches/gperftools-unbreak-memz.patch" \ + "patch -p1 < $TP_DIR/patches/gperftools-osx-arm64.patch" \ "autoreconf -fvi" PROTOBUF_PATCHLEVEL=0 diff --git a/thirdparty/patches/gperftools-osx-arm64.patch b/thirdparty/patches/gperftools-osx-arm64.patch new file mode 100644 index 0000000..7c4cdec --- /dev/null +++ b/thirdparty/patches/gperftools-osx-arm64.patch @@ -0,0 +1,21 @@ +From 23f7072ab6bfa5c0d1b45cf1e8d66053c7b738a8 Mon Sep 17 00:00:00 2001 +From: Grant Henke <[email protected]> +Date: Thu, 26 Nov 2020 20:56:46 -0600 +Subject: [PATCH] Add OS X arm64 program counter + +--- + m4/pc_from_ucontext.m4 | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4 +index 0c7ee281..6e8b5d00 100644 +--- a/m4/pc_from_ucontext.m4 ++++ b/m4/pc_from_ucontext.m4 +@@ -43,6 +43,7 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT], + pc_fields="$pc_fields uc_mcontext->__ss.__rip" # OS X (>=10.5 [untested]) + pc_fields="$pc_fields uc_mcontext->ss.srr0" # OS X (ppc, ppc64 [untested]) + pc_fields="$pc_fields uc_mcontext->__ss.__srr0" # OS X (>=10.5 [untested]) ++ pc_fields="$pc_fields uc_mcontext->__ss.__pc" # OS X (arm64 [untested]) + pc_field_found=false + for pc_field in $pc_fields; do + if ! $pc_field_found; then \ No newline at end of file diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 25abe6a..cb74744 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -50,7 +50,7 @@ GMOCK_VERSION=1.8.0 GMOCK_NAME=googletest-release-$GMOCK_VERSION GMOCK_SOURCE=$TP_SOURCE_DIR/$GMOCK_NAME -GPERFTOOLS_VERSION=2.6.90 +GPERFTOOLS_VERSION=2.8 GPERFTOOLS_NAME=gperftools-$GPERFTOOLS_VERSION GPERFTOOLS_SOURCE=$TP_SOURCE_DIR/$GPERFTOOLS_NAME
