This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new b507fd36e fix(test): fix the failed ASan test for gutil due to
absl::node_hash_map (#2341)
b507fd36e is described below
commit b507fd36e5b96827fce6c9e5332a02d297bd425b
Author: Dan Wang <[email protected]>
AuthorDate: Fri Jan 9 17:18:50 2026 +0800
fix(test): fix the failed ASan test for gutil due to absl::node_hash_map
(#2341)
https://github.com/apache/incubator-pegasus/issues/2344.
The internal types used by `absl::node_hash_map` have different layouts when
compiled with ASan versus non-ASan builds. As a third-party dependency,
Abseil
is always built in non-ASan mode. However, when Pegasus is built with ASan
enabled,
Abseil headers are also compiled under ASan, which leads to a mismatch
between the
header-compiled structures and the Abseil library itself, causing ASAN
tests to fail.
Possible solutions include:
1. When third-party dependencies change, also build those third-party
libraries with
ASan enabled for ASan builds.
2. When building third-party images, additionally produce a separate
ASan-based
image, which can be used by ASan CI workflows when third-party dependencies
themselves have not changed.
---
.../rebuild_thirdparty_if_needed/action.yaml | 2 +-
.github/workflows/lint_and_test_cpp.yaml | 14 +++++--
.github/workflows/lint_and_test_go-client.yml | 1 +
.github/workflows/lint_and_test_java-client.yml | 1 +
.github/workflows/lint_and_test_scala-client.yml | 1 +
.github/workflows/test_nodejs-client.yml | 1 +
.github/workflows/test_python-client.yml | 1 +
run.sh | 4 ++
thirdparty/CMakeLists.txt | 43 ++++++++++++++++++----
9 files changed, 55 insertions(+), 13 deletions(-)
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index 18f2e1081..9b4e7098a 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -38,7 +38,7 @@ runs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1
-DUSE_JEMALLOC=${USE_JEMALLOC} -B build/
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1
-DUSE_JEMALLOC=${USE_JEMALLOC} -DENABLE_ASAN=${ENABLE_ASAN} -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index 311533b1d..6fcf4b8b6 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -60,6 +60,9 @@ jobs:
cpp_clang_tidy_linter:
name: Tidy
runs-on: ubuntu-22.04
+ env:
+ USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
container:
image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
steps:
@@ -86,6 +89,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
container:
image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
steps:
@@ -115,6 +119,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
ARTIFACT_NAME: release
BUILD_OPTIONS: -t release --test
container:
@@ -204,6 +209,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: ON
ARTIFACT_NAME: release_address
BUILD_OPTIONS: --sanitizer address --disable_gperf --test
container:
@@ -259,10 +265,7 @@ jobs:
- dsn_task_tests
- dsn_utils_tests
- dsn.zookeeper.tests
- # TODO(wangdan): absl::node_hash_map will fail when ASan tests run
because some objects
- # have different structures between non-ASan and ASan building. Will
be enabled after
- # this issue is fixed.
- # - gutil_test
+ - gutil_test
# TODO(yingchun): Disable it because we find it's too flaky, we will
re-enable it after
# it has been optimized.
# - partition_split_test
@@ -300,6 +303,7 @@ jobs:
# runs-on: ubuntu-latest
# env:
# USE_JEMALLOC: OFF
+# ENABLE_ASAN: OFF
# ARTIFACT_NAME: release_undefined
# BUILD_OPTIONS: --sanitizer undefined --disable_gperf --test
# container:
@@ -386,6 +390,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: ON
+ ENABLE_ASAN: OFF
ARTIFACT_NAME: release_jemalloc
BUILD_OPTIONS: -t release --use_jemalloc --test
container:
@@ -463,6 +468,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
BUILD_OPTIONS: -t debug --test --separate_servers
PACK_OPTIONS: --separate_servers
container:
diff --git a/.github/workflows/lint_and_test_go-client.yml
b/.github/workflows/lint_and_test_go-client.yml
index 14d1455be..45bae30db 100644
--- a/.github/workflows/lint_and_test_go-client.yml
+++ b/.github/workflows/lint_and_test_go-client.yml
@@ -86,6 +86,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
BUILD_OPTIONS: -t release
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
diff --git a/.github/workflows/lint_and_test_java-client.yml
b/.github/workflows/lint_and_test_java-client.yml
index 48a17be84..0fa681be2 100644
--- a/.github/workflows/lint_and_test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -55,6 +55,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
ARTIFACT_NAME: release_for_java_client
BUILD_OPTIONS: -t release
container:
diff --git a/.github/workflows/lint_and_test_scala-client.yml
b/.github/workflows/lint_and_test_scala-client.yml
index 6e7f74dd5..53fc0612e 100644
--- a/.github/workflows/lint_and_test_scala-client.yml
+++ b/.github/workflows/lint_and_test_scala-client.yml
@@ -63,6 +63,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
BUILD_OPTIONS: -t release
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
diff --git a/.github/workflows/test_nodejs-client.yml
b/.github/workflows/test_nodejs-client.yml
index 6f6d2b671..5abab0fa9 100644
--- a/.github/workflows/test_nodejs-client.yml
+++ b/.github/workflows/test_nodejs-client.yml
@@ -43,6 +43,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
BUILD_OPTIONS: -t release
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
diff --git a/.github/workflows/test_python-client.yml
b/.github/workflows/test_python-client.yml
index c8ae0c69b..9bf45b622 100644
--- a/.github/workflows/test_python-client.yml
+++ b/.github/workflows/test_python-client.yml
@@ -43,6 +43,7 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
+ ENABLE_ASAN: OFF
BUILD_OPTIONS: -t release
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
diff --git a/run.sh b/run.sh
index f63a7bd1b..1c7909577 100755
--- a/run.sh
+++ b/run.sh
@@ -288,6 +288,10 @@ function run_build()
echo "Start building third-parties..."
mkdir -p build
pushd build
+ if [[ "${SANITIZER}" == *address* ]]; then
+ echo "ASan for third-parties is enabled"
+ CMAKE_OPTIONS="${CMAKE_OPTIONS} -DENABLE_ASAN=ON"
+ fi
CMAKE_OPTIONS="${CMAKE_OPTIONS} -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE}"
cmake .. ${CMAKE_OPTIONS}
make -j$JOB_NUM
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index 2077c7cfc..d139262fa 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -57,6 +57,16 @@ message(STATUS "Setting up third-parties...")
file(MAKE_DIRECTORY ${TP_OUTPUT}/include)
file(MAKE_DIRECTORY ${TP_OUTPUT}/lib)
+option(ENABLE_ASAN "enable ASan" OFF)
+message(STATUS "ENABLE_ASAN = ${ENABLE_ASAN}")
+
+set(ASAN_OPTIONS
+ -DCMAKE_C_FLAGS="-fsanitize=address"
+ -DCMAKE_CXX_FLAGS="-fsanitize=address"
+ -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
+ -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address"
+)
+
ExternalProject_Add(boost
URL
https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz
URL_MD5 30a657150d8e5cf431e1de4396a433d0
@@ -119,23 +129,30 @@ ExternalProject_Add(gperftools
DOWNLOAD_NO_PROGRESS true
)
+
+set(ABSL_OPTIONS
+ -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ -DABSL_FIND_GOOGLETEST=OFF
+ -DCMAKE_CXX_STANDARD=17
+)
+
+if (ENABLE_ASAN)
+ list(APPEND ABSL_OPTIONS ${ASAN_OPTIONS})
+endif()
+
ExternalProject_Add(abseil
URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip
https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip
URL_MD5 5c6193dbc82834f8e762c6a28c9cc615
PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_absl_build_on_macos_arm64.patch
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
- -DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DABSL_FIND_GOOGLETEST=OFF
- -DCMAKE_CXX_STANDARD=17
+ CMAKE_ARGS ${ABSL_OPTIONS}
DOWNLOAD_EXTRACT_TIMESTAMP true
DOWNLOAD_NO_PROGRESS true
)
-ExternalProject_Add(protobuf
- URL
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v27.0.tar.gz
- URL_MD5 c96aaf02c8acea549d65bb7b2d549bf6
- CMAKE_ARGS -DCMAKE_BUILD_TYPE=release
+set(PROTOBUF_OPTIONS
+ -DCMAKE_BUILD_TYPE=release
-Dprotobuf_BUILD_TESTS=OFF
-Dprotobuf_BUILD_PROTOC_BINARIES=ON
-Dprotobuf_BUILD_LIBUPB=ON
@@ -149,6 +166,16 @@ ExternalProject_Add(protobuf
-DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+)
+
+if (ENABLE_ASAN)
+ list(APPEND PROTOBUF_OPTIONS ${ASAN_OPTIONS})
+endif()
+
+ExternalProject_Add(protobuf
+ URL
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v27.0.tar.gz
+ URL_MD5 c96aaf02c8acea549d65bb7b2d549bf6
+ CMAKE_ARGS ${PROTOBUF_OPTIONS}
DEPENDS abseil
DOWNLOAD_EXTRACT_TIMESTAMP true
DOWNLOAD_NO_PROGRESS true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
