http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/build_detect_platform ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/build_detect_platform b/thirdparty/rocksdb/build_tools/build_detect_platform new file mode 100755 index 0000000..c7ddb7c --- /dev/null +++ b/thirdparty/rocksdb/build_tools/build_detect_platform @@ -0,0 +1,532 @@ +#!/bin/sh +# +# Detects OS we're compiling on and outputs a file specified by the first +# argument, which in turn gets read while processing Makefile. +# +# The output will set the following variables: +# CC C Compiler path +# CXX C++ Compiler path +# PLATFORM_LDFLAGS Linker flags +# JAVA_LDFLAGS Linker flags for RocksDBJava +# JAVA_STATIC_LDFLAGS Linker flags for RocksDBJava static build +# PLATFORM_SHARED_EXT Extension for shared libraries +# PLATFORM_SHARED_LDFLAGS Flags for building shared library +# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library +# PLATFORM_CCFLAGS C compiler flags +# PLATFORM_CXXFLAGS C++ compiler flags. Will contain: +# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned +# shared libraries, empty otherwise. +# +# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following: +# +# -DROCKSDB_PLATFORM_POSIX if posix-platform based +# -DSNAPPY if the Snappy library is present +# -DLZ4 if the LZ4 library is present +# -DZSTD if the ZSTD library is present +# -DNUMA if the NUMA library is present +# -DTBB if the TBB library is present +# +# Using gflags in rocksdb: +# Our project depends on gflags, which requires users to take some extra steps +# before they can compile the whole repository: +# 1. Install gflags. You may download it from here: +# https://gflags.github.io/gflags/ (Mac users can `brew install gflags`) +# 2. Once installed, add the include path for gflags to your CPATH env var and +# the lib path to LIBRARY_PATH. If installed with default settings, the lib +# will be /usr/local/lib and the include path will be /usr/local/include + +OUTPUT=$1 +if test -z "$OUTPUT"; then + echo "usage: $0 <output-filename>" >&2 + exit 1 +fi + +# we depend on C++11 +PLATFORM_CXXFLAGS="-std=c++11" +# we currently depend on POSIX platform +COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" + +# Default to fbcode gcc on internal fb machines +if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then + FBCODE_BUILD="true" + # If we're compiling with TSAN we need pic build + PIC_BUILD=$COMPILE_WITH_TSAN + if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then + source "$PWD/build_tools/fbcode_config.sh" + else + # we need this to build with MySQL. Don't use for other purposes. + source "$PWD/build_tools/fbcode_config4.8.1.sh" + fi +fi + +# Delete existing output, if it exists +rm -f "$OUTPUT" +touch "$OUTPUT" + +if test -z "$CC"; then + CC=cc +fi + +if test -z "$CXX"; then + CXX=g++ +fi + +# Detect OS +if test -z "$TARGET_OS"; then + TARGET_OS=`uname -s` +fi + +if test -z "$TARGET_ARCHITECTURE"; then + TARGET_ARCHITECTURE=`uname -m` +fi + +if test -z "$CLANG_SCAN_BUILD"; then + CLANG_SCAN_BUILD=scan-build +fi + +if test -z "$CLANG_ANALYZER"; then + CLANG_ANALYZER=$(which clang++ 2> /dev/null) +fi + +COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}" +CROSS_COMPILE= +PLATFORM_CCFLAGS= +PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS" +PLATFORM_SHARED_EXT="so" +PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl," +PLATFORM_SHARED_CFLAGS="-fPIC" +PLATFORM_SHARED_VERSIONED=true + +# generic port files (working on all platform by #ifdef) go directly in /port +GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "` + +# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp +case "$TARGET_OS" in + Darwin) + PLATFORM=OS_MACOSX + COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX" + PLATFORM_SHARED_EXT=dylib + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " + # PORT_FILES=port/darwin/darwin_specific.cc + ;; + IOS) + PLATFORM=IOS + COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE" + PLATFORM_SHARED_EXT=dylib + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " + CROSS_COMPILE=true + PLATFORM_SHARED_VERSIONED= + ;; + Linux) + PLATFORM=OS_LINUX + COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + # PORT_FILES=port/linux/linux_specific.cc + ;; + SunOS) + PLATFORM=OS_SOLARIS + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64" + # PORT_FILES=port/sunos/sunos_specific.cc + ;; + AIX) + PLATFORM=OS_AIX + CC=gcc + COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc" + # PORT_FILES=port/aix/aix_specific.cc + ;; + FreeBSD) + PLATFORM=OS_FREEBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" + # PORT_FILES=port/freebsd/freebsd_specific.cc + ;; + NetBSD) + PLATFORM=OS_NETBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s" + # PORT_FILES=port/netbsd/netbsd_specific.cc + ;; + OpenBSD) + PLATFORM=OS_OPENBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread" + # PORT_FILES=port/openbsd/openbsd_specific.cc + ;; + DragonFly) + PLATFORM=OS_DRAGONFLYBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" + # PORT_FILES=port/dragonfly/dragonfly_specific.cc + ;; + Cygwin) + PLATFORM=CYGWIN + PLATFORM_SHARED_CFLAGS="" + PLATFORM_CXXFLAGS="-std=gnu++11" + COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + # PORT_FILES=port/linux/linux_specific.cc + ;; + OS_ANDROID_CROSSCOMPILE) + PLATFORM=OS_ANDROID + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library + # PORT_FILES=port/android/android.cc + CROSS_COMPILE=true + ;; + *) + echo "Unknown platform!" >&2 + exit 1 +esac + +PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}" +JAVA_LDFLAGS="$PLATFORM_LDFLAGS" +JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS" + +if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then + # Cross-compiling; do not try any compilation tests. + # Also don't need any compilation tests if compiling on fbcode + true +else + if ! test $ROCKSDB_DISABLE_FALLOCATE; then + # Test whether fallocate is available + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <fcntl.h> + #include <linux/falloc.h> + int main() { + int fd = open("/dev/null", 0); + fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT" + fi + fi + + # Test whether Snappy library is installed + # http://code.google.com/p/snappy/ + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <snappy.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy" + fi + + # Test whether gflags library is installed + # http://gflags.github.io/gflags/ + # check if the namespace is gflags + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF + #include <gflags/gflags.h> + using namespace gflags; + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + else + # check if namespace is google + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF + #include <gflags/gflags.h> + using namespace google; + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + fi + fi + + # Test whether zlib library is installed + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <zlib.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DZLIB" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lz" + fi + + # Test whether bzip library is installed + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <bzlib.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DBZIP2" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2" + fi + + # Test whether lz4 library is installed + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <lz4.h> + #include <lz4hc.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DLZ4" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4" + JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4" + fi + + # Test whether zstd library is installed + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <zstd.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DZSTD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd" + fi + + # Test whether numa is available + $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF + #include <numa.h> + #include <numaif.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DNUMA" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma" + fi + + # Test whether tbb is available + $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF + #include <tbb/tbb.h> + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DTBB" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb" + JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb" + fi + + # Test whether jemalloc is available + if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \ + 2>/dev/null; then + # This will enable some preprocessor identifiers in the Makefile + JEMALLOC=1 + # JEMALLOC can be enabled either using the flag (like here) or by + # providing direct link to the jemalloc library + WITH_JEMALLOC_FLAG=1 + else + # jemalloc is not available. Let's try tcmalloc + if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \ + -ltcmalloc 2>/dev/null; then + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc" + JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc" + fi + fi + + # Test whether malloc_usable_size is available + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <malloc.h> + int main() { + size_t res = malloc_usable_size(0); + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE" + fi + + # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <pthread.h> + int main() { + int x = PTHREAD_MUTEX_ADAPTIVE_NP; + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX" + fi + + # Test whether backtrace is available + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <execinfo.h>> + int main() { + void* frames[1]; + backtrace_symbols(frames, backtrace(frames, 1)); + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + else + # Test whether execinfo library is installed + $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <execinfo.h> + int main() { + void* frames[1]; + backtrace_symbols(frames, backtrace(frames, 1)); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo" + fi + fi + + # Test if -pg is supported + $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOF + int main() { + return 0; + } +EOF + if [ "$?" = 0 ]; then + PROFILING_FLAGS=-pg + fi + + # Test whether sync_file_range is supported for compatibility with an old glibc + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <fcntl.h> + int main() { + int fd = open("/dev/null", 0); + sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT" + fi + + # Test whether sched_getcpu is supported + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <sched.h> + int main() { + int cpuid = sched_getcpu(); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT" + fi +fi + +# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning. +# -Wshorten-64-to-32 breaks compilation on FreeBSD i386 +if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then + # Test whether -Wshorten-64-to-32 is available + $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null <<EOF + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32" + fi +fi + +# shall we use HDFS? + +if test "$USE_HDFS"; then + if test -z "$JAVA_HOME"; then + echo "JAVA_HOME has to be set for HDFS usage." + exit 1 + fi + HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS" + HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64" + HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib" + HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm" + COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS" + JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS" +fi + +if test "$USE_SSE"; then + COMMON_FLAGS="$COMMON_FLAGS -msse4.2" +elif test -z "$PORTABLE"; then + if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then + # Tune for this POWER processor, treating '+' models as base models + POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+` + COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER " + elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then + COMMON_FLAGS="$COMMON_FLAGS -march=z10 " + elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then + COMMON_FLAGS="$COMMON_FLAGS -march=native " + fi +fi + +$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #include <cstdint> + #include <nmmintrin.h> + int main() { + volatile uint32_t x = _mm_crc32_u32(0, 0); + } +EOF +if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42" +elif test "$USE_SSE"; then + echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling" +fi + +# iOS doesn't support thread-local storage, but this check would erroneously +# succeed because the cross-compiler flags are added by the Makefile, not this +# script. +if [ "$PLATFORM" != IOS ]; then + $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF + #if defined(_MSC_VER) && !defined(__thread) + #define __thread __declspec(thread) + #endif + int main() { + static __thread int tls; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL" + fi +fi + +PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" +PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" + +VALGRIND_VER="$VALGRIND_VER" + +ROCKSDB_MAJOR=`build_tools/version.sh major` +ROCKSDB_MINOR=`build_tools/version.sh minor` +ROCKSDB_PATCH=`build_tools/version.sh patch` + +echo "CC=$CC" >> "$OUTPUT" +echo "CXX=$CXX" >> "$OUTPUT" +echo "PLATFORM=$PLATFORM" >> "$OUTPUT" +echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT" +echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT" +echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT" +echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT" +echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT" +echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> "$OUTPUT" +echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT" +echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> "$OUTPUT" +echo "JEMALLOC_INCLUDE=$JEMALLOC_INCLUDE" >> "$OUTPUT" +echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT" +echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT" +echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT" +echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT" +echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT" +echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT" +echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" +# This will enable some related identifiers for the preprocessor +if test -n "$JEMALLOC"; then + echo "JEMALLOC=1" >> "$OUTPUT" +fi +# Indicates that jemalloc should be enabled using -ljemalloc flag +# The alternative is to porvide a direct link to the library via JEMALLOC_LIB +# and JEMALLOC_INCLUDE +if test -n "$WITH_JEMALLOC_FLAG"; then + echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT" +fi +echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/cont_integration.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/cont_integration.sh b/thirdparty/rocksdb/build_tools/cont_integration.sh new file mode 100755 index 0000000..06f25c5 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/cont_integration.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2016, Facebook. All rights reserved. +# +# Overall wrapper script for RocksDB continuous builds. The implementation is a +# trivial pulling scheme. We loop infinitely, check if any new changes have been +# committed, if yes then trigger a Sandcastle run, and finally go to sleep again +# for a certain interval. +# + +SRC_GIT_REPO=/data/git/rocksdb-public +error=0 + +function log { + DATE=`date +%Y-%m-%d:%H:%M:%S` + echo $DATE $@ +} + +function log_err { + log "ERROR: $@ Error code: $error." +} + +function update_repo_status { + # Update the parent first. + pushd $SRC_GIT_REPO + + # This is a fatal error. Something in the environment isn't right and we will + # terminate the execution. + error=$? + if [ ! $error -eq 0 ]; then + log_err "Where is $SRC_GIT_REPO?" + exit $error + fi + + HTTPS_PROXY=fwdproxy:8080 git fetch -f + + error=$? + if [ ! $error -eq 0 ]; then + log_err "git fetch -f failed." + popd + return $error + fi + + git update-ref refs/heads/master refs/remotes/origin/master + + error=$? + if [ ! $error -eq 0 ]; then + log_err "git update-ref failed." + popd + return $error + fi + + popd + + # We're back in an instance-specific directory. Get the latest changes. + git pull --rebase + + error=$? + if [ ! $error -eq 0 ]; then + log_err "git pull --rebase failed." + return $error + fi +} + +# +# Execution starts here. +# + +# Path to the determinator from the root of the RocksDB repo. +CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php + +# Value of the previous commit. +PREV_COMMIT= + +log "Starting to monitor for new RocksDB changes ..." +log "Running under `pwd` as `whoami`." + +# Paranoia. Make sure that we're using the right branch. +git checkout master + +error=$? +if [ ! $error -eq 0 ]; then + log_err "This is not good. Can't checkout master. Bye-bye!" + exit 1 +fi + +# We'll run forever and let the execution environment terminate us if we'll +# exceed whatever timeout is set for the job. +while true; +do + # Get the latest changes committed. + update_repo_status + + error=$? + if [ $error -eq 0 ]; then + LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'` + + log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'." + + if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then + log "There were no changes since the last time I checked. Going to sleep." + else + if [ ! -z "$LAST_COMMIT" ]; then + log "New code has been committed or previous commit not known. " \ + "Will trigger the tests." + + PREV_COMMIT=$LAST_COMMIT + log "Updated previous commit to '$PREV_COMMIT'." + + # + # This is where we'll trigger the Sandcastle run. The values for + # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're + # running in. + # + POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR + + error=$? + if [ $error -eq 0 ]; then + log "Sandcastle run successfully triggered." + else + log_err "Failed to trigger Sandcastle run." + fi + else + log_err "Previous commit not updated. Don't know what the last one is." + fi + fi + else + log_err "Getting latest changes failed. Will skip running tests for now." + fi + + # Always sleep, even if errors happens while trying to determine the latest + # commit. This will prevent us terminating in case of transient errors. + log "Will go to sleep for 5 minutes." + sleep 5m +done http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dependencies.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/dependencies.sh b/thirdparty/rocksdb/build_tools/dependencies.sh new file mode 100644 index 0000000..868753b --- /dev/null +++ b/thirdparty/rocksdb/build_tools/dependencies.sh @@ -0,0 +1,18 @@ +GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1 +CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de +LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858 +GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0 +SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787 +ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787 +BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787 +LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787 +ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5 +GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787 +JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b +NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787 +LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb +TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787 +KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e +BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e +VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787 +LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh b/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh new file mode 100644 index 0000000..ef0cda2 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh @@ -0,0 +1,18 @@ +GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc +CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd +LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc +GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc +SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a +ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a +BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a +LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a +ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a +GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a +JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51 +NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a +LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945 +TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a +KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e +BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e +VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a +LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dockerbuild.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/dockerbuild.sh b/thirdparty/rocksdb/build_tools/dockerbuild.sh new file mode 100755 index 0000000..02f6094 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/dockerbuild.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +docker run -v $PWD:/rocks -w /rocks buildpack-deps make http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/error_filter.py ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/error_filter.py b/thirdparty/rocksdb/build_tools/error_filter.py new file mode 100644 index 0000000..9f619cf --- /dev/null +++ b/thirdparty/rocksdb/build_tools/error_filter.py @@ -0,0 +1,167 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +'''Filter for error messages in test output: + - Receives merged stdout/stderr from test on stdin + - Finds patterns of known error messages for test name (first argument) + - Prints those error messages to stdout +''' + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import re +import sys + + +class ErrorParserBase(object): + def parse_error(self, line): + '''Parses a line of test output. If it contains an error, returns a + formatted message describing the error; otherwise, returns None. + Subclasses must override this method. + ''' + raise NotImplementedError + + +class GTestErrorParser(ErrorParserBase): + '''A parser that remembers the last test that began running so it can print + that test's name upon detecting failure. + ''' + _GTEST_NAME_PATTERN = re.compile(r'\[ RUN \] (\S+)$') + # format: '<filename or "unknown file">:<line #>: Failure' + _GTEST_FAIL_PATTERN = re.compile(r'(unknown file|\S+:\d+): Failure$') + + def __init__(self): + self._last_gtest_name = 'Unknown test' + + def parse_error(self, line): + gtest_name_match = self._GTEST_NAME_PATTERN.match(line) + if gtest_name_match: + self._last_gtest_name = gtest_name_match.group(1) + return None + gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line) + if gtest_fail_match: + return '%s failed: %s' % ( + self._last_gtest_name, gtest_fail_match.group(1)) + return None + + +class MatchErrorParser(ErrorParserBase): + '''A simple parser that returns the whole line if it matches the pattern. + ''' + def __init__(self, pattern): + self._pattern = re.compile(pattern) + + def parse_error(self, line): + if self._pattern.match(line): + return line + return None + + +class CompilerErrorParser(MatchErrorParser): + def __init__(self): + # format: '<filename>:<line #>:<column #>: error: <error msg>' + super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:') + + +class ScanBuildErrorParser(MatchErrorParser): + def __init__(self): + super(ScanBuildErrorParser, self).__init__( + r'scan-build: \d+ bugs found.$') + + +class DbCrashErrorParser(MatchErrorParser): + def __init__(self): + super(DbCrashErrorParser, self).__init__(r'\*\*\*.*\^$|TEST FAILED.') + + +class WriteStressErrorParser(MatchErrorParser): + def __init__(self): + super(WriteStressErrorParser, self).__init__( + r'ERROR: write_stress died with exitcode=\d+') + + +class AsanErrorParser(MatchErrorParser): + def __init__(self): + super(AsanErrorParser, self).__init__( + r'==\d+==ERROR: AddressSanitizer:') + + +class UbsanErrorParser(MatchErrorParser): + def __init__(self): + # format: '<filename>:<line #>:<column #>: runtime error: <error msg>' + super(UbsanErrorParser, self).__init__(r'\S+:\d+:\d+: runtime error:') + + +class ValgrindErrorParser(MatchErrorParser): + def __init__(self): + # just grab the summary, valgrind doesn't clearly distinguish errors + # from other log messages. + super(ValgrindErrorParser, self).__init__(r'==\d+== ERROR SUMMARY:') + + +class CompatErrorParser(MatchErrorParser): + def __init__(self): + super(CompatErrorParser, self).__init__(r'==== .*[Ee]rror.* ====$') + + +class TsanErrorParser(MatchErrorParser): + def __init__(self): + super(TsanErrorParser, self).__init__(r'WARNING: ThreadSanitizer:') + + +_TEST_NAME_TO_PARSERS = { + 'punit': [CompilerErrorParser, GTestErrorParser], + 'unit': [CompilerErrorParser, GTestErrorParser], + 'release': [CompilerErrorParser, GTestErrorParser], + 'unit_481': [CompilerErrorParser, GTestErrorParser], + 'release_481': [CompilerErrorParser, GTestErrorParser], + 'clang_unit': [CompilerErrorParser, GTestErrorParser], + 'clang_release': [CompilerErrorParser, GTestErrorParser], + 'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser], + 'code_cov': [CompilerErrorParser, GTestErrorParser], + 'unity': [CompilerErrorParser, GTestErrorParser], + 'lite': [CompilerErrorParser], + 'lite_test': [CompilerErrorParser, GTestErrorParser], + 'stress_crash': [CompilerErrorParser, DbCrashErrorParser], + 'write_stress': [CompilerErrorParser, WriteStressErrorParser], + 'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser], + 'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser], + 'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser], + 'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser], + 'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser], + 'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser], + 'format_compatible': [CompilerErrorParser, CompatErrorParser], + 'run_format_compatible': [CompilerErrorParser, CompatErrorParser], + 'no_compression': [CompilerErrorParser, GTestErrorParser], + 'run_no_compression': [CompilerErrorParser, GTestErrorParser], + 'regression': [CompilerErrorParser], + 'run_regression': [CompilerErrorParser], +} + + +def main(): + if len(sys.argv) != 2: + return 'Usage: %s <test name>' % sys.argv[0] + test_name = sys.argv[1] + if test_name not in _TEST_NAME_TO_PARSERS: + return 'Unknown test name: %s' % test_name + + error_parsers = [] + for parser_cls in _TEST_NAME_TO_PARSERS[test_name]: + error_parsers.append(parser_cls()) + + for line in sys.stdin: + line = line.strip() + for error_parser in error_parsers: + error_msg = error_parser.parse_error(line) + if error_msg is not None: + print(error_msg) + + +if __name__ == '__main__': + sys.exit(main()) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh b/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh new file mode 100755 index 0000000..c087f81 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# fail early +set -e + +if test -z $ROCKSDB_PATH; then + ROCKSDB_PATH=~/rocksdb +fi +source $ROCKSDB_PATH/build_tools/fbcode_config4.8.1.sh + +EXTRA_LDFLAGS="" + +if test -z $ALLOC; then + # default + ALLOC=tcmalloc +elif [[ $ALLOC == "jemalloc" ]]; then + ALLOC=system + EXTRA_LDFLAGS+=" -Wl,--whole-archive $JEMALLOC_LIB -Wl,--no-whole-archive" +fi + +# we need to force mongo to use static library, not shared +STATIC_LIB_DEP_DIR='build/static_library_dependencies' +test -d $STATIC_LIB_DEP_DIR || mkdir $STATIC_LIB_DEP_DIR +test -h $STATIC_LIB_DEP_DIR/`basename $SNAPPY_LIBS` || ln -s $SNAPPY_LIBS $STATIC_LIB_DEP_DIR +test -h $STATIC_LIB_DEP_DIR/`basename $LZ4_LIBS` || ln -s $LZ4_LIBS $STATIC_LIB_DEP_DIR + +EXTRA_LDFLAGS+=" -L $STATIC_LIB_DEP_DIR" + +set -x + +EXTRA_CMD="" +if ! test -e version.json; then + # this is Mongo 3.0 + EXTRA_CMD="--rocksdb \ + --variant-dir=linux2/norm + --cxx=${CXX} \ + --cc=${CC} \ + --use-system-zlib" # add this line back to normal code path + # when https://jira.mongodb.org/browse/SERVER-19123 is resolved +fi + +scons \ + LINKFLAGS="$EXTRA_LDFLAGS $EXEC_LDFLAGS $PLATFORM_LDFLAGS" \ + CCFLAGS="$CXXFLAGS -L $STATIC_LIB_DEP_DIR" \ + LIBS="lz4 gcc stdc++" \ + LIBPATH="$ROCKSDB_PATH" \ + CPPPATH="$ROCKSDB_PATH/include" \ + -j32 \ + --allocator=$ALLOC \ + --nostrip \ + --opt=on \ + --disable-minimum-compiler-version-enforcement \ + --use-system-snappy \ + --disable-warnings-as-errors \ + $EXTRA_CMD $* http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fbcode_config.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/fbcode_config.sh b/thirdparty/rocksdb/build_tools/fbcode_config.sh new file mode 100644 index 0000000..b8609a1 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/fbcode_config.sh @@ -0,0 +1,156 @@ +#!/bin/sh +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ and clang compilers and also +# uses jemalloc +# Environment variables that change the behavior of this script: +# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies.sh" + +CFLAGS="" + +# libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" + +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" + +# snappy +SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" +if test -z $PIC_BUILD; then + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" +else + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" +fi +CFLAGS+=" -DSNAPPY" + +if test -z $PIC_BUILD; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" + CFLAGS+=" -DZLIB" + + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" + CFLAGS+=" -DBZIP2" + + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" + CFLAGS+=" -DLZ4" + + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" + CFLAGS+=" -DZSTD" +fi + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +if test -z $PIC_BUILD; then + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" +else + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" +fi +CFLAGS+=" -DGFLAGS=gflags" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" + +if test -z $PIC_BUILD; then + # location of numa + NUMA_INCLUDE=" -I $NUMA_BASE/include/" + NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" + CFLAGS+=" -DNUMA" + + # location of libunwind + LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" +fi + +# location of TBB +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +if test -z $PIC_BUILD; then + TBB_LIBS="$TBB_BASE/lib/libtbb.a" +else + TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a" +fi +CFLAGS+=" -DTBB" + +# use Intel SSE support for checksum calculations +export USE_SSE=1 + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +CLANG_BIN="$CLANG_BASE/bin" +CLANG_LIB="$CLANG_BASE/lib" +CLANG_SRC="$CLANG_BASE/../../src" + +CLANG_ANALYZER="$CLANG_BIN/clang++" +CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + + CFLAGS+=" -B$BINUTILS/gold" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + + KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" + + CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + CFLAGS+=" -Wno-expansion-to-defined " + CXXFLAGS="-nostdinc++" +fi + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS" +EXEC_LDFLAGS+=" -B$BINUTILS/gold" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +LUA_PATH="$LUA_BASE" + +if test -z $PIC_BUILD; then + LUA_LIB=" $LUA_PATH/lib/liblua.a" +else + LUA_LIB=" $LUA_PATH/lib/liblua_pic.a" +fi + +export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh b/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh new file mode 100644 index 0000000..f5b8334 --- /dev/null +++ b/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ compiler and also +# uses jemalloc + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies_4.8.1.sh" + +# location of libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" + +# location of glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" + +# location of snappy headers and libraries +SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include" +SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" + +# location of zlib headers and libraries +ZLIB_INCLUDE=" -I $ZLIB_BASE/include" +ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" + +# location of bzip headers and libraries +BZIP2_INCLUDE=" -I $BZIP2_BASE/include/" +BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a" + +LZ4_INCLUDE=" -I $LZ4_BASE/include" +LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" + +ZSTD_INCLUDE=" -I $ZSTD_BASE/include" +ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include" +JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a" + +# location of numa +NUMA_INCLUDE=" -I $NUMA_BASE/include/" +NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" + +# location of libunwind +LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" + +# location of tbb +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +TBB_LIBS="$TBB_BASE/lib/libtbb.a" + +# use Intel SSE support for checksum calculations +export USE_SSE=1 + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + + CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_BIN="$CLANG_BASE/bin" + CLANG_LIB="$CLANG_BASE/lib" + CLANG_INCLUDE="$CLANG_LIB/clang/*/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + + KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/" + + CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + CXXFLAGS="-nostdinc++" +fi + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42" +CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +LUA_PATH="$LUA_BASE" + +export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/format-diff.sh ---------------------------------------------------------------------- diff --git a/thirdparty/rocksdb/build_tools/format-diff.sh b/thirdparty/rocksdb/build_tools/format-diff.sh new file mode 100755 index 0000000..81221ed --- /dev/null +++ b/thirdparty/rocksdb/build_tools/format-diff.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# If clang_format_diff.py command is not specfied, we assume we are able to +# access directly without any path. +if [ -z $CLANG_FORMAT_DIFF ] +then +CLANG_FORMAT_DIFF="clang-format-diff.py" +fi + +# Check clang-format-diff.py +if ! which $CLANG_FORMAT_DIFF &> /dev/null +then + echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!" + echo "You can download clang-format-diff.py by running: " + echo " curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}" + echo "You can download clang-format by running: " + echo " brew install clang-format" + echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}" + exit 128 +fi + +# Check argparse, a library that clang-format-diff.py requires. +python 2>/dev/null << EOF +import argparse +EOF + +if [ "$?" != 0 ] +then + echo "To run clang-format-diff.py, we'll need the library "argparse" to be" + echo "installed. You can try either of the follow ways to install it:" + echo " 1. Manually download argparse: https://pypi.python.org/pypi/argparse"; + echo " 2. easy_install argparse (if you have easy_install)" + echo " 3. pip install argparse (if you have pip)" + exit 129 +fi + +# TODO(kailiu) following work is not complete since we still need to figure +# out how to add the modified files done pre-commit hook to git's commit index. +# +# Check if this script has already been added to pre-commit hook. +# Will suggest user to add this script to pre-commit hook if their pre-commit +# is empty. +# PRE_COMMIT_SCRIPT_PATH="`git rev-parse --show-toplevel`/.git/hooks/pre-commit" +# if ! ls $PRE_COMMIT_SCRIPT_PATH &> /dev/null +# then +# echo "Would you like to add this script to pre-commit hook, which will do " +# echo -n "the format check for all the affected lines before you check in (y/n):" +# read add_to_hook +# if [ "$add_to_hook" == "y" ] +# then +# ln -s `git rev-parse --show-toplevel`/build_tools/format-diff.sh $PRE_COMMIT_SCRIPT_PATH +# fi +# fi +set -e + +uncommitted_code=`git diff HEAD` +LAST_MASTER=`git merge-base master HEAD` + +# If there's no uncommitted changes, we assume user are doing post-commit +# format check, in which case we'll check the modified lines since last commit +# from master. Otherwise, we'll check format of the uncommitted code only. +if [ -z "$uncommitted_code" ] +then + # Check the format of last commit + diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1) +else + # Check the format of uncommitted lines, + diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1) +fi + +if [ -z "$diffs" ] +then + echo "Nothing needs to be reformatted!" + exit 0 +fi + +# Highlight the insertion/deletion from the clang-format-diff.py's output +COLOR_END="\033[0m" +COLOR_RED="\033[0;31m" +COLOR_GREEN="\033[0;32m" + +echo -e "Detect lines that doesn't follow the format rules:\r" +# Add the color to the diff. lines added will be green; lines removed will be red. +echo "$diffs" | + sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" | + sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/" + +if [[ "$OPT" == *"-DTRAVIS"* ]] +then + exit 1 +fi + +echo -e "Would you like to fix the format automatically (y/n): \c" + +# Make sure under any mode, we can read user input. +exec < /dev/tty +read to_fix + +if [ "$to_fix" != "y" ] +then + exit 1 +fi + +# Do in-place format adjustment. +if [ -z "$uncommitted_code" ] +then + git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1 +else + git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1 +fi +echo "Files reformatted!" + +# Amend to last commit if user do the post-commit format check +if [ -z "$uncommitted_code" ]; then + echo -e "Would you like to amend the changes to last commit (`git log HEAD --oneline | head -1`)? (y/n): \c" + read to_amend + + if [ "$to_amend" == "y" ] + then + git commit -a --amend --reuse-message HEAD + echo "Amended to last commit" + fi +fi
