This is an automated email from the ASF dual-hosted git repository.
guangmingchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git
The following commit(s) were added to refs/heads/master by this push:
new 7a7d1c83 Support AddressSanitizer (#2890)
7a7d1c83 is described below
commit 7a7d1c83e54952302126f00263196ee77d35ba8f
Author: Bright Chen <chenguangmin...@foxmail.com>
AuthorDate: Wed Mar 26 20:07:45 2025 +0800
Support AddressSanitizer (#2890)
* Support AddressSanitizer
* Add gperftools helper header
---
.github/workflows/ci-linux.yml | 30 +++++--
BUILD.bazel | 8 +-
CMakeLists.txt | 5 ++
bazel/config/BUILD.bazel | 6 ++
config_brpc.sh | 18 ++++-
docs/cn/getting_started.md | 2 -
docs/cn/sanitizers.md | 32 ++++++++
docs/en/getting_started.md | 2 -
src/brpc/details/http_message.h | 2 +-
src/brpc/details/http_parser.cpp | 4 +-
.../policy/consistent_hashing_load_balancer.cpp | 5 +-
src/brpc/policy/hasher.cpp | 6 +-
src/brpc/policy/http_rpc_protocol.cpp | 2 +-
src/brpc/socket.cpp | 4 +-
src/bthread/bthread.cpp | 9 ++-
src/bthread/execution_queue.cpp | 1 -
src/bthread/execution_queue_inl.h | 12 ++-
src/bthread/stack.cpp | 2 +-
src/bthread/stack.h | 5 +-
src/bthread/stack_inl.h | 92 +++++++++++++++++++++-
src/bthread/task_control.cpp | 5 +-
src/bthread/task_group.cpp | 68 +++++++++++++++-
src/bthread/task_group.h | 7 +-
src/bthread/task_group_inl.h | 8 +-
src/bthread/task_tracer.cpp | 2 +
src/butil/compiler_specific.h | 15 +++-
src/butil/debug/address_annotations.h | 42 ++++++++++
src/butil/debug/leak_annotations.h | 8 +-
src/butil/logging.cc | 9 ++-
src/butil/object_pool.h | 5 ++
src/butil/object_pool_inl.h | 70 ++++++++++++----
test/BUILD.bazel | 10 +++
test/Makefile | 2 +-
test/brpc_builtin_service_unittest.cpp | 8 +-
test/brpc_event_dispatcher_unittest.cpp | 6 +-
test/brpc_h2_unsent_message_unittest.cpp | 2 +-
test/brpc_http_rpc_protocol_unittest.cpp | 24 ++++--
test/brpc_hulu_pbrpc_protocol_unittest.cpp | 1 -
test/brpc_input_messenger_unittest.cpp | 7 +-
test/brpc_load_balancer_unittest.cpp | 2 +-
test/brpc_protobuf_json_unittest.cpp | 20 ++---
test/brpc_snappy_compress_unittest.cpp | 2 +-
test/brpc_socket_unittest.cpp | 2 +-
test/bthread_cond_unittest.cpp | 2 +-
test/bthread_dispatcher_unittest.cpp | 5 +-
test/bthread_execution_queue_unittest.cpp | 2 +-
test/bthread_fd_unittest.cpp | 5 +-
test/bthread_mutex_unittest.cpp | 4 +-
test/bthread_rwlock_unittest.cpp | 3 +-
test/bthread_setconcurrency_unittest.cpp | 4 +-
test/bthread_unittest.cpp | 14 +++-
test/bthread_work_stealing_queue_unittest.cpp | 2 +
test/bvar_lock_timer_unittest.cpp | 4 +-
test/gperftools_helper.h | 34 ++++++++
test/iobuf_unittest.cpp | 4 +
test/logging_unittest.cc | 4 +-
test/mpsc_queue_unittest.cc | 3 -
test/popen_unittest.cpp | 1 +
test/run_tests.sh | 5 +-
test/security_unittest.cc | 5 ++
test/stack_trace_unittest.cc | 1 +
test/thread_key_unittest.cpp | 2 -
62 files changed, 554 insertions(+), 122 deletions(-)
diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml
index f87d6519..64dedb6d 100644
--- a/.github/workflows/ci-linux.yml
+++ b/.github/workflows/ci-linux.yml
@@ -61,7 +61,7 @@ jobs:
- uses: ./.github/actions/install-all-dependences
- uses: ./.github/actions/init-make-config
with:
- options: --cc=gcc --cxx=g++ --with-thrift --with-glog --with-rdma
--with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer
--werror
+ options: --cc=gcc --cxx=g++ --with-thrift --with-glog --with-rdma
--with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer
--werror --with-asan
- name: compile
run: |
make -j ${{env.proc_num}}
@@ -76,7 +76,7 @@ jobs:
export CC=gcc && export CXX=g++
mkdir build
cd build
- cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON
-DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON
-WITH_BTHREAD_TRACER=ON ..
+ cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON
-DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON
-DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON ..
- name: compile
run: |
cd build
@@ -86,7 +86,7 @@ jobs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- - run: bazel build --verbose_failures --define with_mesalink=false
--define with_glog=true --define with_thrift=true --define
with_debug_bthread_sche_safety=true --define with_debug_lock=true -- //...
-//example/...
+ - run: bazel build --verbose_failures --define with_mesalink=false
--define with_glog=true --define with_thrift=true --define
with_debug_bthread_sche_safety=true --define with_debug_lock=true --define
with_asan=true -- //... -//example/...
clang-compile-with-make:
runs-on: ubuntu-22.04
@@ -135,7 +135,7 @@ jobs:
- uses: ./.github/actions/install-all-dependences
- uses: ./.github/actions/init-make-config
with:
- options: --cc=clang --cxx=clang++ --with-thrift --with-glog
--with-rdma --with-debug-bthread-sche-safety --with-debug-lock
--with-bthread-tracer --werror
+ options: --cc=clang --cxx=clang++ --with-thrift --with-glog
--with-rdma --with-debug-bthread-sche-safety --with-debug-lock
--with-bthread-tracer --werror --with-asan
- name: compile
run: |
make -j ${{env.proc_num}}
@@ -150,7 +150,7 @@ jobs:
export CC=clang && export CXX=clang++
mkdir build
cd build
- cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON
-DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON
-WITH_BTHREAD_TRACER=ON ..
+ cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON
-DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON
-DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON ..
- name: compile
run: |
cd build
@@ -160,7 +160,7 @@ jobs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- - run: bazel build --verbose_failures --action_env=CC=clang --define
with_mesalink=false --define with_glog=true --define with_thrift=true --define
with_debug_bthread_sche_safety=true --define with_debug_lock=true -- //...
-//example/...
+ - run: bazel build --verbose_failures --action_env=CC=clang --define
with_mesalink=false --define with_glog=true --define with_thrift=true --define
with_debug_bthread_sche_safety=true --define with_debug_lock=true --define
with_asan=true -- //... -//example/...
clang-unittest:
runs-on: ubuntu-22.04
@@ -179,3 +179,21 @@ jobs:
run: |
cd test
sh ./run_tests.sh
+
+ clang-unittest-asan:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v2
+ - uses: ./.github/actions/install-essential-dependences
+ - uses: ./.github/actions/init-ut-make-config
+ with:
+ options: --cc=clang-12 --cxx=clang++-12 --with-bthread-tracer
--with-asan
+ - name: compile tests
+ run: |
+ cat config.mk
+ cd test
+ make NEED_GPERFTOOLS=0 -j ${{env.proc_num}}
+ - name: run tests
+ run: |
+ cd test
+ sh ./run_tests.sh
diff --git a/BUILD.bazel b/BUILD.bazel
index 610d01d0..4e0217fc 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -53,6 +53,9 @@ COPTS = [
}) + select({
"@platforms//cpu:x86_64": ["-DBRPC_BTHREAD_TRACER"],
"//conditions:default": [],
+}) + select({
+ "//bazel/config:brpc_with_asan": ["-fsanitize=address"],
+ "//conditions:default": [""],
})
LINKOPTS = [
@@ -87,7 +90,10 @@ LINKOPTS = [
"-libverbs",
],
"//conditions:default": [],
-})
+}) + select({
+ "//bazel/config:brpc_with_asan": ["-fsanitize=address"],
+ "//conditions:default": [""],
+ })
genrule(
name = "config_h",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 19b92ba5..639d0d87 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,7 @@ option(WITH_SNAPPY "With snappy" OFF)
option(WITH_RDMA "With RDMA" OFF)
option(WITH_DEBUG_BTHREAD_SCHE_SAFETY "With debugging bthread sche safety" OFF)
option(WITH_DEBUG_LOCK "With debugging lock" OFF)
+option(WITH_ASAN "With AddressSanitizer" OFF)
option(BUILD_UNIT_TESTS "Whether to build unit tests" OFF)
option(BUILD_FUZZ_TESTS "Whether to build fuzz tests" OFF)
option(BUILD_BRPC_TOOLS "Whether to build brpc tools" ON)
@@ -134,6 +135,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
endif()
set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} ${DEFINE_CLOCK_GETTIME}
-DBRPC_WITH_GLOG=${WITH_GLOG_VAL} -DBRPC_WITH_RDMA=${WITH_RDMA_VAL}
-DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=${WITH_DEBUG_BTHREAD_SCHE_SAFETY_VAL}
-DBRPC_DEBUG_LOCK=${WITH_DEBUG_LOCK_VAL}")
+if (WITH_ASAN)
+ set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} -fsanitize=address")
+ set(CMAKE_C_FLAGS "${CMAKE_CPP_FLAGS} -fsanitize=address")
+endif()
if(WITH_MESALINK)
set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} -DUSE_MESALINK")
endif()
diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel
index 10965747..57569dd3 100644
--- a/bazel/config/BUILD.bazel
+++ b/bazel/config/BUILD.bazel
@@ -120,4 +120,10 @@ config_setting(
name = "brpc_with_debug_lock",
define_values = {"with_debug_lock": "true"},
visibility = ["//visibility:public"],
+)
+
+config_setting(
+ name = "brpc_with_asan",
+ define_values = {"with_asan": "true"},
+ visibility = ["//visibility:public"],
)
\ No newline at end of file
diff --git a/config_brpc.sh b/config_brpc.sh
index 25f506fe..a16bab81 100755
--- a/config_brpc.sh
+++ b/config_brpc.sh
@@ -38,12 +38,13 @@ else
LDD=ldd
fi
-TEMP=`getopt -o v: --long
headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,nodebugsymbols,werror
-n 'config_brpc' -- "$@"`
+TEMP=`getopt -o v: --long
headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,nodebugsymbols,werror
-n 'config_brpc' -- "$@"`
WITH_GLOG=0
WITH_THRIFT=0
WITH_RDMA=0
WITH_MESALINK=0
WITH_BTHREAD_TRACER=0
+WITH_ASAN=0
BRPC_DEBUG_BTHREAD_SCHE_SAFETY=0
DEBUGSYMBOLS=-g
WERROR=
@@ -74,6 +75,7 @@ while true; do
--with-bthread-tracer) WITH_BTHREAD_TRACER=1; shift 1 ;;
--with-debug-bthread-sche-safety ) BRPC_DEBUG_BTHREAD_SCHE_SAFETY=1;
shift 1 ;;
--with-debug-lock ) BRPC_DEBUG_LOCK=1; shift 1 ;;
+ --with-asan) WITH_ASAN=1; shift 1 ;;
--nodebugsymbols ) DEBUGSYMBOLS=; shift 1 ;;
--werror ) WERROR=-Werror; shift 1 ;;
-- ) shift; break ;;
@@ -345,10 +347,15 @@ else
CXXFLAGS="-std=c++0x"
fi
-LEVELDB_HDR=$(find_dir_of_header_or_die leveldb/db.h)
-
CPPFLAGS=
+if [ $WITH_ASAN != 0 ]; then
+ CPPFLAGS="${CPPFLAGS} -fsanitize=address"
+ DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -fsanitize=address"
+fi
+
+LEVELDB_HDR=$(find_dir_of_header_or_die leveldb/db.h)
+
if [ $WITH_BTHREAD_TRACER != 0 ]; then
if [ "$SYSTEM" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
>&2 $ECHO "bthread tracer is only supported on Linux x86_64 platform"
@@ -425,7 +432,7 @@ append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS"
append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS"
# CPP means C PreProcessing, not C PlusPlus
-CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG
-DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY
-DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK"
+CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG
-DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY
-DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK"
# Avoid over-optimizations of TLS variables by GCC>=4.8
# See: https://github.com/apache/brpc/issues/1693
@@ -523,7 +530,10 @@ append_to_output "ifeq (\$(NEED_GPERFTOOLS), 1)"
TCMALLOC_LIB=$(find_dir_of_lib tcmalloc_and_profiler)
if [ -z "$TCMALLOC_LIB" ]; then
append_to_output " \$(error \"Fail to find gperftools\")"
+elif [ $WITH_ASAN != 0 ]; then
+ append_to_output " \$(error \"gperftools is not compatible with ASAN\")"
else
+ append_to_output " CPPFLAGS+=-DBRPC_ENABLE_CPU_PROFILER"
append_to_output_libs "$TCMALLOC_LIB" " "
if [ -f $TCMALLOC_LIB/libtcmalloc.$SO ]; then
append_to_output " DYNAMIC_LINKINGS+=-ltcmalloc_and_profiler"
diff --git a/docs/cn/getting_started.md b/docs/cn/getting_started.md
index 76aa6f86..1b91e15b 100644
--- a/docs/cn/getting_started.md
+++ b/docs/cn/getting_started.md
@@ -352,8 +352,6 @@ bRPC 中使用了 protobuf 内部 API，上游不保证相关 API 的兼容性
[1.8.0](https://github.com/apache/brpc/releases/tag/1.8.0) 中
[#2406](https://github.com/apache/brpc/pull/2406) 和
[#2493](https://github.com/apache/brpc/pull/2493)引入了部分 proto3 语法，所以目前 bRPC 不再兼容
protobuf 2.x 版本。如果你希望使用 2.x 版本，可以使用 1.8.0 之前的 bRPC 版本。
-pb 3.x中的Arena至今没被支持。
-
## gflags: 2.1-2.2.2
2.1.1
中存在一处已知问题，需要[补丁](https://github.com/gflags/gflags/commit/408061b46974cc8377a8a794a048ecae359ad887)。
diff --git a/docs/cn/sanitizers.md b/docs/cn/sanitizers.md
new file mode 100644
index 00000000..e902e101
--- /dev/null
+++ b/docs/cn/sanitizers.md
@@ -0,0 +1,32 @@
+# Sanitizers
+
+新版本的GCC/Clang支持[sanitizers](https://github.com/google/sanitizers)，方便开发者排查代码中的bug。
bRPC对sanitizers提供了一定的支持。
+
+## AddressSanitizer(ASan)
+
+ASan提供了[对协程的支持](https://reviews.llvm.org/D20913)。
在bthread创建、切换、销毁时，让ASan知道当前bthread的栈信息，主要用于维护[fake
stack](https://github.com/google/sanitizers/wiki/AddressSanitizerUseAfterReturn)。
+
+bRPC中启用ASan的方法：给config_brpc.sh增加`--with-glog`选项、给cmake增加`-DWITH_GLOG=ON`选项或者给bazel增加`--define
with_asan=true`选项。
+
+另外需要注意的是，ASan没法检测非ASan分配内存或者对象池复用内存。所以我们封装了两个宏，让ASan知道内存块是否能被使用。在非ASan环境下，这两个宏什么也不做，没有开销。
+
+```c++
+#include <butil/debug/address_annotations.h>
+
+BUTIL_ASAN_POISON_MEMORY_REGION(addr, size);
+BUTIL_ASAN_UNPOISON_MEMORY_REGION(addr, size);
+```
+
+其他问题：如果ASan报告中new/delete的调用栈不完整，可以通过设置`fast_unwind_on_malloc=0`回溯出完整的调用栈了。需要注意的是`fast_unwind_on_malloc=0`很耗性能。
+
+## ThreadSanitizer(TSan)
+
+待支持（TODO）
+
+## LeakSanitizer(LSan) / MemorySanitizer(MSan) /
UndefinedBehaviorSanitizer(UBSan)
+
+bRPC默认支持这三种sanitizers，编译及链接时加上对应的选项即可启用对应的sanitizers：
+
+1. LSan: `-fsanitize=leak`；
+2. MSan: `-fsanitize=memory`；
+3. UBSan: `-fsanitize=undefined`；
diff --git a/docs/en/getting_started.md b/docs/en/getting_started.md
index a2ae80d4..b2363ce3 100644
--- a/docs/en/getting_started.md
+++ b/docs/en/getting_started.md
@@ -347,8 +347,6 @@ Please [submit
issue](https://github.com/apache/brpc/issues) if you have any pro
[#2406](https://github.com/apache/brpc/pull/2406) and
[#2493](https://github.com/apache/brpc/pull/2493) in [version
1.8.0]((https://github.com/apache/brpc/releases/tag/1.8.0)) introduce some
proto3 syntax, so currently bRPC is no longer compatible with pb 2.x version.
If you want to use pb 2.x version, you can use bRPC version before 1.8.0.
-Arena in pb 3.x is not supported yet.
-
## gflags: 2.0-2.2.2
[gflags
patch](https://github.com/gflags/gflags/commit/408061b46974cc8377a8a794a048ecae359ad887)
is required when compiled with 2.1.1.
diff --git a/src/brpc/details/http_message.h b/src/brpc/details/http_message.h
index 6ff2d031..655ba1a9 100644
--- a/src/brpc/details/http_message.h
+++ b/src/brpc/details/http_message.h
@@ -88,7 +88,7 @@ public:
bool read_body_progressively() const { return _read_body_progressively; }
void set_read_body_progressively(bool read_body_progressively) {
- this->_read_body_progressively = read_body_progressively;
+ _read_body_progressively = read_body_progressively;
}
// Send new parts of the body to the reader. If the body already has some
diff --git a/src/brpc/details/http_parser.cpp b/src/brpc/details/http_parser.cpp
index 0f9e906a..a4aa4d12 100644
--- a/src/brpc/details/http_parser.cpp
+++ b/src/brpc/details/http_parser.cpp
@@ -2191,13 +2191,13 @@ http_parser_init (http_parser *parser, enum
http_parser_type t)
const char *
http_errno_name(enum http_errno err) {
- assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ assert(err <
(http_errno)(sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
return http_strerror_tab[err].name;
}
const char *
http_errno_description(enum http_errno err) {
- assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ assert(err <
(http_errno)(sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
return http_strerror_tab[err].description;
}
diff --git a/src/brpc/policy/consistent_hashing_load_balancer.cpp
b/src/brpc/policy/consistent_hashing_load_balancer.cpp
index 19c8849c..c18c9a34 100644
--- a/src/brpc/policy/consistent_hashing_load_balancer.cpp
+++ b/src/brpc/policy/consistent_hashing_load_balancer.cpp
@@ -19,6 +19,7 @@
#include <algorithm> //
std::set_union
#include <array>
#include <gflags/gflags.h>
+#include <openssl/md5.h>
#include "butil/containers/flat_map.h"
#include "butil/errno.h"
#include "butil/strings/string_number_conversions.h"
@@ -102,10 +103,10 @@ bool KetamaReplicaPolicy::Build(ServerId server,
CHECK(num_replicas % points_per_hash == 0)
<< "Ketam hash replicas number(" << num_replicas << ") should be n*4";
for (size_t i = 0; i < num_replicas / points_per_hash; ++i) {
- char host[32];
+ char host[256];
int len = snprintf(host, sizeof(host), "%s-%lu",
endpoint2str(ptr->remote_side()).c_str(), i);
- unsigned char digest[16];
+ unsigned char digest[MD5_DIGEST_LENGTH];
MD5HashSignature(host, len, digest);
for (size_t j = 0; j < points_per_hash; ++j) {
ConsistentHashingLoadBalancer::Node node;
diff --git a/src/brpc/policy/hasher.cpp b/src/brpc/policy/hasher.cpp
index f533099a..ba33eb54 100644
--- a/src/brpc/policy/hasher.cpp
+++ b/src/brpc/policy/hasher.cpp
@@ -28,12 +28,12 @@ namespace policy {
void MD5HashSignature(const void* key, size_t len, unsigned char* results) {
MD5_CTX my_md5;
MD5_Init(&my_md5);
- MD5_Update(&my_md5, (const unsigned char *)key, len);
+ MD5_Update(&my_md5, key, len);
MD5_Final(results, &my_md5);
}
uint32_t MD5Hash32(const void* key, size_t len) {
- unsigned char results[16];
+ unsigned char results[MD5_DIGEST_LENGTH];
MD5HashSignature(key, len, results);
return ((uint32_t) (results[3] & 0xFF) << 24)
| ((uint32_t) (results[2] & 0xFF) << 16)
@@ -48,7 +48,7 @@ uint32_t MD5Hash32V(const butil::StringPiece* keys, size_t
num_keys) {
MD5_Update(&ctx, (const unsigned char *)keys[i].data(),
keys[i].size());
}
- unsigned char results[16];
+ unsigned char results[MD5_DIGEST_LENGTH];
MD5_Final(results, &ctx);
return ((uint32_t) (results[3] & 0xFF) << 24)
| ((uint32_t) (results[2] & 0xFF) << 16)
diff --git a/src/brpc/policy/http_rpc_protocol.cpp
b/src/brpc/policy/http_rpc_protocol.cpp
index 07fc7da8..a1353b14 100644
--- a/src/brpc/policy/http_rpc_protocol.cpp
+++ b/src/brpc/policy/http_rpc_protocol.cpp
@@ -1708,7 +1708,7 @@ void HttpContext::CheckProgressiveRead(const void* arg,
Socket *socket) {
header().uri().path(), (Server *)arg,
const_cast<std::string *>(&header().unresolved_path()));
if (sp != NULL && sp->params.enable_progressive_read) {
- this->set_read_body_progressively(true);
+ set_read_body_progressively(true);
socket->read_will_be_progressive(CONNECTION_TYPE_SHORT);
}
}
diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp
index 056a9fca..5e969fc8 100644
--- a/src/brpc/socket.cpp
+++ b/src/brpc/socket.cpp
@@ -508,9 +508,11 @@ void
Socket::ReturnSuccessfulWriteRequest(Socket::WriteRequest* p) {
DCHECK(p->data.empty());
AddOutputMessages(1);
const bthread_id_t id_wait = p->id_wait;
+ bool is_notify_on_success = p->is_notify_on_success();
butil::return_object(p);
+ // Do not access `p' after it is returned to ObjectPool.
if (id_wait != INVALID_BTHREAD_ID) {
- if (p->is_notify_on_success() && !Failed()) {
+ if (is_notify_on_success && !Failed()) {
bthread_id_error(id_wait, 0);
} else {
NotifyOnFailed(id_wait);
diff --git a/src/bthread/bthread.cpp b/src/bthread/bthread.cpp
index e124f4bc..8d4e2026 100644
--- a/src/bthread/bthread.cpp
+++ b/src/bthread/bthread.cpp
@@ -350,7 +350,7 @@ bthread_t bthread_self(void) {
bthread::TaskGroup* g = bthread::tls_task_group;
// note: return 0 for main tasks now, which include main thread and
// all work threads. So that we can identify main tasks from logs
- // more easily. This is probably questionable in future.
+ // more easily. This is probably questionable in the future.
if (g != NULL && !g->is_current_main_task()/*note*/) {
return g->current_tid();
}
@@ -361,6 +361,13 @@ int bthread_equal(bthread_t t1, bthread_t t2) {
return t1 == t2;
}
+#ifdef BUTIL_USE_ASAN
+// Fixme!!!
+// The noreturn `bthread_exit' may cause a warning of ASan, but does not abort
the program.
+//
+// ==94463==WARNING: ASan is ignoring requested __asan_handle_no_return: stack
type: default top: 0x00016dd7f000; bottom 0x00010b1a4000; size: 0x000062bdb000
(1656598528)
+// False positive error reports may follow
+#endif // BUTIL_USE_ASAN
void bthread_exit(void* retval) {
bthread::TaskGroup* g = bthread::tls_task_group;
if (g != NULL && !g->is_current_main_task()) {
diff --git a/src/bthread/execution_queue.cpp b/src/bthread/execution_queue.cpp
index bb01882c..88d96c2b 100644
--- a/src/bthread/execution_queue.cpp
+++ b/src/bthread/execution_queue.cpp
@@ -22,7 +22,6 @@
#include "bthread/execution_queue.h"
#include "butil/memory/singleton_on_pthread_once.h"
-#include "butil/object_pool.h" // butil::get_object
#include "butil/resource_pool.h" // butil::get_resource
#include "butil/threading/platform_thread.h"
diff --git a/src/bthread/execution_queue_inl.h
b/src/bthread/execution_queue_inl.h
index f5998a26..cf7d2ee5 100644
--- a/src/bthread/execution_queue_inl.h
+++ b/src/bthread/execution_queue_inl.h
@@ -27,8 +27,9 @@
#include "butil/memory/scoped_ptr.h" // butil::scoped_ptr
#include "butil/logging.h" // LOG
#include "butil/time.h" // butil::cpuwide_time_ns
-#include "bvar/bvar.h" // bvar::Adder
-#include "bthread/butex.h" // butex_construct
+#include "butil/object_pool.h" // butil::get_object
+#include "bvar/bvar.h" // bvar::Adder
+#include "bthread/butex.h" // butex_construct
#include "butil/synchronization/condition_variable.h"
namespace bthread {
@@ -585,4 +586,11 @@ inline int ExecutionQueueBase::dereference() {
} // namespace bthread
+namespace butil {
+// `TaskNode::cancel' may access the TaskNode object returned to the
ObjectPool<TaskNode>,
+// so ObjectPool<TaskNode> can not poison the memory region of TaskNode.
+template <>
+struct ObjectPoolWithASanPoison<bthread::TaskNode> : false_type {};
+} // namespace butil
+
#endif //BTHREAD_EXECUTION_QUEUE_INL_H
diff --git a/src/bthread/stack.cpp b/src/bthread/stack.cpp
index c312ef83..71daf9d7 100644
--- a/src/bthread/stack.cpp
+++ b/src/bthread/stack.cpp
@@ -139,7 +139,7 @@ void deallocate_stack_storage(StackStorage* s) {
return;
}
s_stack_count.fetch_sub(1, butil::memory_order_relaxed);
- if (s->guardsize <= 0) {
+ if (s->guardsize == 0) {
free((char*)s->bottom - memsize);
} else {
munmap((char*)s->bottom - memsize, memsize);
diff --git a/src/bthread/stack.h b/src/bthread/stack.h
index 46cafb4a..91d1df60 100644
--- a/src/bthread/stack.h
+++ b/src/bthread/stack.h
@@ -31,8 +31,8 @@
namespace bthread {
struct StackStorage {
- int stacksize;
- int guardsize;
+ unsigned stacksize;
+ unsigned guardsize;
// Assume stack grows upwards.
//
http://www.boost.org/doc/libs/1_55_0/libs/context/doc/html/context/stack.html
void* bottom;
@@ -62,6 +62,7 @@ enum StackType {
};
struct ContextualStack {
+ virtual ~ContextualStack() = default;
bthread_fcontext_t context;
StackType stacktype;
StackStorage storage;
diff --git a/src/bthread/stack_inl.h b/src/bthread/stack_inl.h
index 868775d4..d765dfd0 100644
--- a/src/bthread/stack_inl.h
+++ b/src/bthread/stack_inl.h
@@ -28,6 +28,83 @@ DECLARE_int32(tc_stack_normal);
namespace bthread {
+#ifdef BUTIL_USE_ASAN
+namespace internal {
+
+BUTIL_FORCE_INLINE void ASanPoisonMemoryRegion(const StackStorage& storage) {
+ if (NULL == storage.bottom) {
+ return;
+ }
+
+ CHECK_GT((void*)storage.bottom,
+ reinterpret_cast<void*>(storage.stacksize + + storage.guardsize));
+ BUTIL_ASAN_POISON_MEMORY_REGION(
+ (char*)storage.bottom - storage.stacksize, storage.stacksize);
+}
+
+BUTIL_FORCE_INLINE void ASanUnpoisonMemoryRegion(const StackStorage& storage) {
+ if (NULL == storage.bottom) {
+ return;
+ }
+ CHECK_GT(storage.bottom,
+ reinterpret_cast<void*>(storage.stacksize + storage.guardsize));
+ BUTIL_ASAN_UNPOISON_MEMORY_REGION(
+ (char*)storage.bottom - storage.stacksize, storage.stacksize);
+}
+
+
+BUTIL_FORCE_INLINE void StartSwitchFiber(void** fake_stack_save, StackStorage&
storage) {
+ if (NULL == storage.bottom) {
+ return;
+ }
+ RELEASE_ASSERT(storage.bottom >
+ reinterpret_cast<void*>(storage.stacksize +
storage.guardsize));
+ // Lowest address of this stack.
+ void* asan_stack_bottom = (char*)storage.bottom - storage.stacksize;
+ BUTIL_ASAN_START_SWITCH_FIBER(fake_stack_save, asan_stack_bottom,
storage.stacksize);
+}
+
+BUTIL_FORCE_INLINE void FinishSwitchFiber(void* fake_stack_save) {
+ BUTIL_ASAN_FINISH_SWITCH_FIBER(fake_stack_save, NULL, NULL);
+}
+
+class ScopedASanFiberSwitcher {
+public:
+ ScopedASanFiberSwitcher(StackStorage& storage, bool ending) {
+ // If bthread will be quit here, pass NULL as `fake_stack_save',
+ // so that ASan knows it can destroy the fake stack.
+ StartSwitchFiber(ending ? NULL : &_fake_stack, storage);
+ }
+
+ ~ScopedASanFiberSwitcher() {
+ FinishSwitchFiber(_fake_stack);
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(ScopedASanFiberSwitcher);
+
+private:
+ void* _fake_stack{NULL};
+};
+
+#define BTHREAD_ASAN_POISON_MEMORY_REGION(storage) \
+ ::bthread::internal::ASanPoisonMemoryRegion(storage)
+
+#define BTHREAD_ASAN_UNPOISON_MEMORY_REGION(storage) \
+ ::bthread::internal::ASanUnpoisonMemoryRegion(storage)
+
+#define BTHREAD_SCOPED_ASAN_FIBER_SWITCHER(storage, ending) \
+ ::bthread::internal::ScopedASanFiberSwitcher switcher(storage, ending)
+
+} // namespace internal
+#else
+
+// If ASan are used, the annotations should be no-ops.
+#define BTHREAD_ASAN_POISON_MEMORY_REGION(storage) ((void)(storage))
+#define BTHREAD_ASAN_UNPOISON_MEMORY_REGION(storage) ((void)(storage))
+#define BTHREAD_SCOPED_ASAN_FIBER_SWITCHER(storage, ending) ((void)(storage),
(void)(ending))
+
+#endif // BUTIL_USE_ASAN
+
struct MainStackClass {};
struct SmallStackClass {
@@ -57,10 +134,14 @@ template <typename StackClass> struct StackFactory {
}
context = bthread_make_fcontext(storage.bottom, storage.stacksize,
entry);
stacktype = (StackType)StackClass::stacktype;
+ // It's poisoned prior to use.
+ BTHREAD_ASAN_POISON_MEMORY_REGION(storage);
}
~Wrapper() {
if (context) {
context = NULL;
+ // Unpoison to avoid affecting other allocator.
+ BTHREAD_ASAN_UNPOISON_MEMORY_REGION(storage);
deallocate_stack_storage(&storage);
storage.zeroize();
}
@@ -68,11 +149,16 @@ template <typename StackClass> struct StackFactory {
};
static ContextualStack* get_stack(void (*entry)(intptr_t)) {
- return butil::get_object<Wrapper>(entry);
+ ContextualStack* cs = butil::get_object<Wrapper>(entry);
+ // Marks stack as addressable.
+ BTHREAD_ASAN_UNPOISON_MEMORY_REGION(cs->storage);
+ return cs;
}
- static void return_stack(ContextualStack* sc) {
- butil::return_object(static_cast<Wrapper*>(sc));
+ static void return_stack(ContextualStack* cs) {
+ // Marks stack as unaddressable.
+ BTHREAD_ASAN_POISON_MEMORY_REGION(cs->storage);
+ butil::return_object(static_cast<Wrapper*>(cs));
}
};
diff --git a/src/bthread/task_control.cpp b/src/bthread/task_control.cpp
index 55ed1f2e..545bc322 100644
--- a/src/bthread/task_control.cpp
+++ b/src/bthread/task_control.cpp
@@ -228,7 +228,7 @@ int TaskControl::init(int concurrency) {
const int rc = pthread_create(&_workers[i], NULL, worker_thread, arg);
if (rc) {
delete arg;
- LOG(ERROR) << "Fail to create _workers[" << i << "], " <<
berror(rc);
+ PLOG(ERROR) << "Fail to create _workers[" << i << "]";
return -1;
}
}
@@ -272,8 +272,7 @@ int TaskControl::add_workers(int num, bthread_tag_t tag) {
&_workers[i + old_concurency], NULL, worker_thread, arg);
if (rc) {
delete arg;
- LOG(WARNING) << "Fail to create _workers[" << i + old_concurency
- << "], " << berror(rc);
+ PLOG(WARNING) << "Fail to create _workers[" << i + old_concurency
<< "]";
_concurrency.fetch_sub(1, butil::memory_order_release);
break;
}
diff --git a/src/bthread/task_group.cpp b/src/bthread/task_group.cpp
index 361efa59..0c924ac3 100644
--- a/src/bthread/task_group.cpp
+++ b/src/bthread/task_group.cpp
@@ -208,12 +208,39 @@ TaskGroup::~TaskGroup() {
if (_main_tid) {
TaskMeta* m = address_meta(_main_tid);
CHECK(_main_stack == m->stack);
+#ifdef BUTIL_USE_ASAN
+ _main_stack->storage.bottom = NULL;
+ _main_stack->storage.stacksize = 0;
+#endif // BUTIL_USE_ASAN
return_stack(m->release_stack());
return_resource(get_slot(_main_tid));
_main_tid = 0;
}
}
+#ifdef BUTIL_USE_ASAN
+int PthreadAttrGetStack(void*& stack_addr, size_t& stack_size) {
+#if defined(OS_MACOSX)
+ stack_addr = pthread_get_stackaddr_np(pthread_self());
+ stack_size = pthread_get_stacksize_np(pthread_self());
+ return 0;
+#else
+ pthread_attr_t attr;
+ int rc = pthread_getattr_np(pthread_self(), &attr);
+ if (0 != rc) {
+ LOG(ERROR) << "Fail to get pthread attributes: " << berror(rc);
+ return rc;
+ }
+ rc = pthread_attr_getstack(&attr, &stack_addr, &stack_size);
+ if (0 != rc) {
+ LOG(ERROR) << "Fail to get pthread stack: " << berror(rc);
+ }
+ pthread_attr_destroy(&attr);
+ return rc;
+#endif // OS_MACOSX
+}
+#endif // BUTIL_USE_ASAN
+
int TaskGroup::init(size_t runqueue_capacity) {
if (_rq.init(runqueue_capacity) != 0) {
LOG(FATAL) << "Fail to init _rq";
@@ -223,6 +250,15 @@ int TaskGroup::init(size_t runqueue_capacity) {
LOG(FATAL) << "Fail to init _remote_rq";
return -1;
}
+
+#ifdef BUTIL_USE_ASAN
+ void* stack_addr = NULL;
+ size_t stack_size = 0;
+ if (0 != PthreadAttrGetStack(stack_addr, stack_size)) {
+ return -1;
+ }
+#endif // BUTIL_USE_ASAN
+
ContextualStack* stk = get_stack(STACK_TYPE_MAIN, NULL);
if (NULL == stk) {
LOG(FATAL) << "Fail to get main stack container";
@@ -247,6 +283,12 @@ int TaskGroup::init(size_t runqueue_capacity) {
m->tid = make_tid(*m->version_butex, slot);
m->set_stack(stk);
+#ifdef BUTIL_USE_ASAN
+ stk->storage.bottom = stack_addr;
+ stk->storage.stacksize = stack_size;
+ // No guard size required for ASan.
+#endif // BUTIL_USE_ASAN
+
_cur_meta = m;
_main_tid = m->tid;
_main_stack = stk;
@@ -255,6 +297,15 @@ int TaskGroup::init(size_t runqueue_capacity) {
return 0;
}
+#ifdef BUTIL_USE_ASAN
+void TaskGroup::asan_task_runner(intptr_t) {
+ // This is a new thread, and it doesn't have the fake stack yet. ASan will
+ // create it lazily, for now just pass NULL.
+ internal::FinishSwitchFiber(NULL);
+ task_runner(0);
+}
+#endif // BUTIL_USE_ASAN
+
void TaskGroup::task_runner(intptr_t skip_remained) {
// NOTE: tls_task_group is volatile since tasks are moved around
// different groups.
@@ -564,11 +615,18 @@ void TaskGroup::ending_sched(TaskGroup** pg) {
TaskMeta* next_meta = address_meta(next_tid);
if (next_meta->stack == NULL) {
if (next_meta->stack_type() == cur_meta->stack_type()) {
+ // Reuse the stack of the current ending task.
+ //
// also works with pthread_task scheduling to pthread_task, the
// transfered stack is just _main_stack.
next_meta->set_stack(cur_meta->release_stack());
} else {
+#ifdef BUTIL_USE_ASAN
+ ContextualStack* stk = get_stack(
+ next_meta->stack_type(), asan_task_runner);
+#else
ContextualStack* stk = get_stack(next_meta->stack_type(),
task_runner);
+#endif // BUTIL_USE_ASAN
if (stk) {
next_meta->set_stack(stk);
} else {
@@ -581,7 +639,7 @@ void TaskGroup::ending_sched(TaskGroup** pg) {
}
}
}
- sched_to(pg, next_meta);
+ sched_to(pg, next_meta, true);
}
void TaskGroup::sched(TaskGroup** pg) {
@@ -602,7 +660,7 @@ void TaskGroup::sched(TaskGroup** pg) {
extern void CheckBthreadScheSafety();
-void TaskGroup::sched_to(TaskGroup** pg, TaskMeta* next_meta) {
+void TaskGroup::sched_to(TaskGroup** pg, TaskMeta* next_meta, bool cur_ending)
{
TaskGroup* g = *pg;
#ifndef NDEBUG
if ((++g->_sched_recursive_guard) > 1) {
@@ -657,7 +715,11 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta*
next_meta) {
g->_control->_task_tracer.set_status(TASK_STATUS_JUMPING,
cur_meta);
g->_control->_task_tracer.set_status(TASK_STATUS_JUMPING,
next_meta);
#endif // BRPC_BTHREAD_TRACER
- jump_stack(cur_meta->stack, next_meta->stack);
+ {
+ BTHREAD_SCOPED_ASAN_FIBER_SWITCHER(
+ cur_meta->stack->storage, cur_ending);
+ jump_stack(cur_meta->stack, next_meta->stack);
+ }
// probably went to another group, need to assign g again.
g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
#ifdef BRPC_BTHREAD_TRACER
diff --git a/src/bthread/task_group.h b/src/bthread/task_group.h
index c3d2ae46..488ad492 100644
--- a/src/bthread/task_group.h
+++ b/src/bthread/task_group.h
@@ -81,7 +81,7 @@ public:
// Suspend caller and run bthread `next_tid' in TaskGroup *pg.
// Purpose of this function is to avoid pushing `next_tid' to _rq and
// then being popped by sched(pg), which is not necessary.
- static void sched_to(TaskGroup** pg, TaskMeta* next_meta);
+ static void sched_to(TaskGroup** pg, TaskMeta* next_meta, bool cur_ending);
static void sched_to(TaskGroup** pg, bthread_t next_tid);
static void exchange(TaskGroup** pg, TaskMeta* next_meta);
@@ -204,7 +204,7 @@ private:
friend class TaskControl;
// You shall use TaskControl::create_group to create new instance.
- explicit TaskGroup(TaskControl*);
+ explicit TaskGroup(TaskControl* c);
int init(size_t runqueue_capacity);
@@ -212,6 +212,9 @@ friend class TaskControl;
// of groups are postponed to avoid race.
~TaskGroup();
+#ifdef BUTIL_USE_ASAN
+ static void asan_task_runner(intptr_t);
+#endif // BUTIL_USE_ASAN
static void task_runner(intptr_t skip_remained);
// Callbacks for set_remained()
diff --git a/src/bthread/task_group_inl.h b/src/bthread/task_group_inl.h
index 75c377e1..4842bf69 100644
--- a/src/bthread/task_group_inl.h
+++ b/src/bthread/task_group_inl.h
@@ -56,13 +56,17 @@ inline void TaskGroup::exchange(TaskGroup** pg, TaskMeta*
next_meta) {
? ready_to_run_in_worker_ignoresignal
: ready_to_run_in_worker),
&args);
- TaskGroup::sched_to(pg, next_meta);
+ TaskGroup::sched_to(pg, next_meta, false);
}
inline void TaskGroup::sched_to(TaskGroup** pg, bthread_t next_tid) {
TaskMeta* next_meta = address_meta(next_tid);
if (next_meta->stack == NULL) {
+#ifdef BUTIL_USE_ASAN
+ ContextualStack* stk = get_stack(next_meta->stack_type(),
asan_task_runner);
+#else
ContextualStack* stk = get_stack(next_meta->stack_type(), task_runner);
+#endif // BUTIL_USE_ASAN
if (stk) {
next_meta->set_stack(stk);
} else {
@@ -75,7 +79,7 @@ inline void TaskGroup::sched_to(TaskGroup** pg, bthread_t
next_tid) {
}
}
// Update now_ns only when wait_task did yield.
- sched_to(pg, next_meta);
+ sched_to(pg, next_meta, false);
}
inline void TaskGroup::push_rq(bthread_t tid) {
diff --git a/src/bthread/task_tracer.cpp b/src/bthread/task_tracer.cpp
index acdf9208..af88ba64 100644
--- a/src/bthread/task_tracer.cpp
+++ b/src/bthread/task_tracer.cpp
@@ -305,6 +305,8 @@ TaskTracer::Result TaskTracer::TraceImpl(bthread_t tid) {
return result;
}
+// Instruct ASan to ignore this function.
+BUTIL_ATTRIBUTE_NO_SANITIZE_ADDRESS
unw_cursor_t TaskTracer::MakeCursor(bthread_fcontext_t fcontext) {
unw_cursor_t cursor;
unw_init_local(&cursor, &_context);
diff --git a/src/butil/compiler_specific.h b/src/butil/compiler_specific.h
index 924fe439..7c1c2662 100644
--- a/src/butil/compiler_specific.h
+++ b/src/butil/compiler_specific.h
@@ -208,10 +208,21 @@
#endif
// Instruct ASan is enabled.
-#if BUTIL_HAS_FEATURE(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
-#define BUTIL_USE_ASAN 1
+#if defined(BUTIL_USE_ASAN)
+#error "BUTIL_USE_ASAN cannot be set directly."
+#elif BUTIL_HAS_FEATURE(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
+#define BUTIL_USE_ASAN
#endif
+//
https://github.com/google/sanitizers/wiki/AddressSanitizer#turning-off-instrumentation
+// Attribute to instruct ASan to ignore a function.
+#if defined(COMPILER_GCC)
+# define BUTIL_ATTRIBUTE_NO_SANITIZE_ADDRESS
__attribute__((no_sanitize_address))
+#else
+# define BUTIL_ATTRIBUTE_NO_SANITIZE_ADDRESS
+#endif
+
+
// Tell the compiler a function is using a printf-style format string.
// |format_param| is the one-based index of the format string parameter;
// |dots_param| is the one-based index of the "..." parameter.
diff --git a/src/butil/debug/address_annotations.h
b/src/butil/debug/address_annotations.h
new file mode 100644
index 00000000..81e7ab11
--- /dev/null
+++ b/src/butil/debug/address_annotations.h
@@ -0,0 +1,42 @@
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BUTIL_DEBUG_ADDRESS_ANNOTATIONS_H_
+#define BUTIL_DEBUG_ADDRESS_ANNOTATIONS_H_
+
+#include "butil/macros.h"
+
+// It provides AddressSanitizer annotations for bthread and memory management.
+// See <sanitizer/asan_interface.h> for detail of these annotations.
+
+#ifdef BUTIL_USE_ASAN
+
+#include <sanitizer/asan_interface.h>
+
+#define BUTIL_ASAN_POISON_MEMORY_REGION(addr, size) \
+ __asan_poison_memory_region(addr, size)
+
+#define BUTIL_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ __asan_unpoison_memory_region(addr, size)
+
+#define BUTIL_ASAN_ADDRESS_IS_POISONED(addr) \
+ __asan_address_is_poisoned(addr)
+
+#define BUTIL_ASAN_START_SWITCH_FIBER(fake_stack_save, bottom, size) \
+ __sanitizer_start_switch_fiber(fake_stack_save, bottom, size)
+
+#define BUTIL_ASAN_FINISH_SWITCH_FIBER(fake_stack_save, bottom_old, size_old) \
+ __sanitizer_finish_switch_fiber(fake_stack_save, bottom_old, size_old)
+
+#else
+// If ASan is not used, these annotations are no-ops.
+#define BUTIL_ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr),
(void)(size))
+#define BUTIL_ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr),
(void)(size))
+#define BUTIL_ASAN_START_SWITCH_FIBER(fake_stack_save, bottom, size) \
+ ((void)(fake_stack_save), (void)(bottom), (void)(size))
+#define BUTIL_ASAN_FINISH_SWITCH_FIBER(fake_stack_save, bottom_old, size_old) \
+ ((void)(fake_stack_save), (void)(bottom_old), (void)(size_old))
+#endif // BUTIL_USE_ASAN
+
+#endif // BUTIL_DEBUG_ADDRESS_ANNOTATIONS_H_
diff --git a/src/butil/debug/leak_annotations.h
b/src/butil/debug/leak_annotations.h
index 231e9982..47387acb 100644
--- a/src/butil/debug/leak_annotations.h
+++ b/src/butil/debug/leak_annotations.h
@@ -32,23 +32,23 @@ void __lsan_do_leak_check();
} // extern "C"
class ScopedLeakSanitizerDisabler {
- public:
+public:
ScopedLeakSanitizerDisabler() { __lsan_disable(); }
~ScopedLeakSanitizerDisabler() { __lsan_enable(); }
- private:
+private:
DISALLOW_COPY_AND_ASSIGN(ScopedLeakSanitizerDisabler);
};
#define ANNOTATE_SCOPED_MEMORY_LEAK \
ScopedLeakSanitizerDisabler leak_sanitizer_disabler; static_cast<void>(0)
-#define ANNOTATE_LEAKING_OBJECT_PTR(X) __lsan_ignore_object(X);
+#define ANNOTATE_LEAKING_OBJECT_PTR(X) __lsan_ignore_object(X)
#else
// If neither HeapChecker nor LSan are used, the annotations should be no-ops.
#define ANNOTATE_SCOPED_MEMORY_LEAK ((void)0)
-#define ANNOTATE_LEAKING_OBJECT_PTR(X) ((void)0)
+#define ANNOTATE_LEAKING_OBJECT_PTR(X) ((void)(X))
#endif
diff --git a/src/butil/logging.cc b/src/butil/logging.cc
index a15251b2..9b09af6d 100644
--- a/src/butil/logging.cc
+++ b/src/butil/logging.cc
@@ -464,6 +464,13 @@ TimeVal GetTimestamp() {
}
struct BAIDU_CACHELINE_ALIGNMENT LogInfo {
+ ~LogInfo() = default;
+ void clear() {
+ file.clear();
+ func.clear();
+ content.clear();
+ }
+
std::string file;
std::string func;
std::string content;
@@ -751,7 +758,7 @@ bool AsyncLogger::IsLogComplete(LogRequest* old_head) {
void AsyncLogger::DoLog(LogRequest* req) {
DoLog(req->log_info);
- req->log_info.content.clear();
+ req->log_info.clear();
}
void AsyncLogger::DoLog(const LogInfo& log_info) {
diff --git a/src/butil/object_pool.h b/src/butil/object_pool.h
index aa265dfa..92bc4036 100644
--- a/src/butil/object_pool.h
+++ b/src/butil/object_pool.h
@@ -23,6 +23,7 @@
#define BUTIL_OBJECT_POOL_H
#include <cstddef> // size_t
+#include "butil/type_traits.h"
// ObjectPool is a derivative class of ResourcePool to allocate and
// reuse fixed-size objects without identifiers.
@@ -63,6 +64,10 @@ template <typename T> struct ObjectPoolValidator {
static bool validate(const T*) { return true; }
};
+//
+template <typename T>
+struct ObjectPoolWithASanPoison : true_type {};
+
} // namespace butil
#include "butil/object_pool_inl.h"
diff --git a/src/butil/object_pool_inl.h b/src/butil/object_pool_inl.h
index f4853262..20d4aea5 100644
--- a/src/butil/object_pool_inl.h
+++ b/src/butil/object_pool_inl.h
@@ -22,15 +22,16 @@
#ifndef BUTIL_OBJECT_POOL_INL_H
#define BUTIL_OBJECT_POOL_INL_H
-#include <iostream> // std::ostream
-#include <pthread.h> // pthread_mutex_t
-#include <algorithm> // std::max, std::min
-#include "butil/atomicops.h" // butil::atomic
-#include "butil/macros.h" // BAIDU_CACHELINE_ALIGNMENT
-#include "butil/scoped_lock.h" // BAIDU_SCOPED_LOCK
-#include "butil/thread_local.h" // BAIDU_THREAD_LOCAL
-#include "butil/memory/aligned_memory.h" // butil::AlignedMemory
+#include <iostream> // std::ostream
+#include <pthread.h> // pthread_mutex_t
+#include <algorithm> // std::max, std::min
#include <vector>
+#include "butil/atomicops.h" // butil::atomic
+#include "butil/macros.h" // BAIDU_CACHELINE_ALIGNMENT
+#include "butil/scoped_lock.h" // BAIDU_SCOPED_LOCK
+#include "butil/thread_local.h" // BAIDU_THREAD_LOCAL
+#include "butil/memory/aligned_memory.h" // butil::AlignedMemory
+#include "butil/debug/address_annotations.h"
#ifdef BUTIL_OBJECT_POOL_NEED_FREE_ITEM_NUM
#define BAIDU_OBJECT_POOL_FREE_ITEM_NUM_ADD1 \
@@ -54,7 +55,7 @@ template <typename T>
struct ObjectPoolFreeChunk<T, 0> {
size_t nfree;
T* ptrs[0];
-};
+};
struct ObjectPoolInfo {
size_t local_pool_num;
@@ -85,6 +86,29 @@ public:
template <typename T>
class BAIDU_CACHELINE_ALIGNMENT ObjectPool {
+private:
+#ifdef BUTIL_USE_ASAN
+ static void asan_poison_memory_region(T* ptr) {
+ if (!ObjectPoolWithASanPoison<T>::value || NULL == ptr) {
+ return;
+ }
+ // Marks the object as addressable.
+ BUTIL_ASAN_POISON_MEMORY_REGION(ptr, sizeof(T));
+ }
+ static void asan_unpoison_memory_region(T* ptr) {
+ if (!ObjectPoolWithASanPoison<T>::value || NULL == ptr) {
+ return;
+ }
+ // Marks the object as unaddressable.
+ BUTIL_ASAN_UNPOISON_MEMORY_REGION(ptr, sizeof(T));
+ }
+#define OBJECT_POOL_ASAN_POISON_MEMORY_REGION(ptr)
asan_poison_memory_region(ptr)
+#define OBJECT_POOL_ASAN_UNPOISON_MEMORY_REGION(ptr)
asan_unpoison_memory_region(ptr)
+#else
+#define OBJECT_POOL_ASAN_POISON_MEMORY_REGION(ptr) ((void)(ptr))
+#define OBJECT_POOL_ASAN_UNPOISON_MEMORY_REGION(ptr) ((void)(ptr))
+#endif // BUTIL_USE_ASAN
+
public:
static const size_t BLOCK_NITEM = ObjectPoolBlockItemNum<T>::value;
static const size_t FREE_CHUNK_NITEM = BLOCK_NITEM;
@@ -93,8 +117,15 @@ public:
// global list(_free_chunks).
typedef ObjectPoolFreeChunk<T, FREE_CHUNK_NITEM> FreeChunk;
typedef ObjectPoolFreeChunk<T, 0> DynamicFreeChunk;
-
+#ifdef BUTIL_USE_ASAN
+ // According to
https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning ,
+ // The allocated chunks should start with 8-aligned addresses,
+ // so that AlignedMemory starts with at least 8-aligned addresses.
+ typedef AlignedMemory<sizeof(T), __alignof__(T) < 8 ? 8 : __alignof__(T)>
BlockItem;
+#else
typedef AlignedMemory<sizeof(T), __alignof__(T)> BlockItem;
+#endif
+
// When a thread needs memory, it allocates a Block. To improve locality,
// items in the Block are only used by the thread.
// To support cache-aligned objects, align Block.items by cacheline.
@@ -169,6 +200,8 @@ public:
obj->~T(); \
return NULL; \
} \
+ /* It's poisoned prior to use. */ \
+ OBJECT_POOL_ASAN_POISON_MEMORY_REGION(obj); \
++_cur_block->nitem; \
return obj; \
} \
@@ -181,6 +214,8 @@ public:
obj->~T(); \
return NULL; \
} \
+ /* It's poisoned prior to use. */ \
+ OBJECT_POOL_ASAN_POISON_MEMORY_REGION(obj); \
++_cur_block->nitem; \
return obj; \
} \
@@ -199,6 +234,9 @@ public:
#undef BAIDU_OBJECT_POOL_GET
inline int return_object(T* ptr) {
+ // TODO. Refer to ASan to implement a efficient quarantine
mechanism.
+ OBJECT_POOL_ASAN_POISON_MEMORY_REGION(ptr);
+
// Return to local free list
if (_cur_free.nfree < ObjectPool::free_chunk_nitem()) {
_cur_free.ptrs[_cur_free.nfree++] = ptr;
@@ -238,10 +276,12 @@ public:
template <typename... Args>
inline T* get_object(Args&&... args) {
LocalPool* lp = get_or_new_local_pool();
+ T* ptr = NULL;
if (BAIDU_LIKELY(lp != NULL)) {
- return lp->get(std::forward<Args>(args)...);
+ ptr = lp->get(std::forward<Args>(args)...);
+ OBJECT_POOL_ASAN_UNPOISON_MEMORY_REGION(ptr);
}
- return NULL;
+ return ptr;
}
inline int return_object(T* ptr) {
@@ -432,8 +472,10 @@ private:
continue;
}
for (size_t k = 0; k < b->nitem; ++k) {
- T* const objs = (T*)b->items;
- objs[k].~T();
+ T* obj = (T*)&b->items[k];
+ // Unpoison to avoid affecting other allocator.
+ OBJECT_POOL_ASAN_UNPOISON_MEMORY_REGION(obj);
+ obj->~T();
}
delete b;
}
diff --git a/test/BUILD.bazel b/test/BUILD.bazel
index 1fd937b2..05420ae3 100644
--- a/test/BUILD.bazel
+++ b/test/BUILD.bazel
@@ -177,6 +177,13 @@ cc_library(
],
)
+cc_library(
+ name = "gperftools_helper",
+ hdrs = [
+ "gperftools_helper.h",
+ ],
+)
+
cc_test(
name = "butil_test",
srcs = TEST_BUTIL_SOURCES + [
@@ -188,6 +195,7 @@ cc_test(
deps = [
":cc_test_proto",
":sstream_workaround",
+ ":gperftools_helper",
"//:brpc",
"@com_google_googletest//:gtest",
],
@@ -208,6 +216,7 @@ cc_test(
copts = COPTS,
deps = [
":sstream_workaround",
+ ":gperftools_helper",
"//:bvar",
"@com_google_googletest//:gtest",
],
@@ -236,6 +245,7 @@ cc_test(
copts = COPTS,
deps = [
":sstream_workaround",
+ ":gperftools_helper",
"//:brpc",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
diff --git a/test/Makefile b/test/Makefile
index 874e6928..0f1534b7 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -19,7 +19,7 @@ NEED_GPERFTOOLS=1
NEED_GTEST=1
include ../config.mk
CPPFLAGS+=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D_GNU_SOURCE -DUSE_SYMBOLIZE
-DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS
-D__STDC_CONSTANT_MACROS -DUNIT_TEST -Dprivate=public -Dprotected=public
-DBVAR_NOT_LINK_DEFAULT_VARIABLES --include sstream_workaround.h
-CXXFLAGS=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing
-Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x
+CXXFLAGS+=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing
-Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x
#required by butil/crc32.cc to boost performance for 10x
ifeq ($(shell test $(GCC_VERSION) -ge 40400; echo $$?),0)
diff --git a/test/brpc_builtin_service_unittest.cpp
b/test/brpc_builtin_service_unittest.cpp
index 8ea17a47..638e5d4c 100644
--- a/test/brpc_builtin_service_unittest.cpp
+++ b/test/brpc_builtin_service_unittest.cpp
@@ -25,7 +25,7 @@
#include <gtest/gtest.h>
#include <gflags/gflags.h>
#include <google/protobuf/descriptor.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/time.h"
#include "butil/macros.h"
#include "brpc/socket.h"
@@ -113,7 +113,7 @@ void MyVLogSite() {
void CheckContent(const brpc::Controller& cntl, const char* name) {
const std::string& content = cntl.response_attachment().to_string();
std::size_t pos = content.find(name);
- ASSERT_TRUE(pos != std::string::npos) << "name=" << name;
+ ASSERT_TRUE(pos != std::string::npos) << "name=" << name << " content=" <<
content;
}
void CheckErrorText(const brpc::Controller& cntl, const char* error) {
@@ -717,6 +717,7 @@ TEST_F(BuiltinServiceTest, rpcz) {
}
}
+#if defined(BRPC_ENABLE_CPU_PROFILER) || defined(BAIDU_RPC_ENABLE_CPU_PROFILER)
TEST_F(BuiltinServiceTest, pprof) {
brpc::PProfService service;
{
@@ -758,6 +759,7 @@ TEST_F(BuiltinServiceTest, pprof) {
CheckContent(cntl, "brpc_builtin_service_unittest");
}
}
+#endif // BRPC_ENABLE_CPU_PROFILER || BAIDU_RPC_ENABLE_CPU_PROFILER
TEST_F(BuiltinServiceTest, dir) {
brpc::DirService service;
@@ -933,6 +935,7 @@ TEST_F(BuiltinServiceTest, sockets) {
}
}
+#if defined(BRPC_ENABLE_CPU_PROFILER) || defined(BAIDU_RPC_ENABLE_CPU_PROFILER)
TEST_F(BuiltinServiceTest, memory) {
brpc::MemoryService service;
brpc::MemoryRequest req;
@@ -950,3 +953,4 @@ TEST_F(BuiltinServiceTest, memory) {
CheckContent(cntl, "tcmalloc.pageheap_free_bytes");
CheckContent(cntl, "tcmalloc.pageheap_unmapped_bytes");
}
+#endif // BRPC_ENABLE_CPU_PROFILER || BAIDU_RPC_ENABLE_CPU_PROFILER
diff --git a/test/brpc_event_dispatcher_unittest.cpp
b/test/brpc_event_dispatcher_unittest.cpp
index 185e9f2d..5c0aa064 100644
--- a/test/brpc_event_dispatcher_unittest.cpp
+++ b/test/brpc_event_dispatcher_unittest.cpp
@@ -23,7 +23,7 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <gtest/gtest.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/fd_utility.h"
@@ -345,13 +345,11 @@ TEST_F(EventDispatcherTest, dispatch_tasks) {
ProfilerStart("event_dispatcher.prof");
butil::Timer tm;
tm.start();
-
sleep(5);
-
tm.stop();
ProfilerStop();
LOG(INFO) << "End profiling";
-
+
size_t client_bytes = 0;
size_t server_bytes = 0;
for (size_t i = 0; i < NCLIENT; ++i) {
diff --git a/test/brpc_h2_unsent_message_unittest.cpp
b/test/brpc_h2_unsent_message_unittest.cpp
index 79b56f55..acb79039 100644
--- a/test/brpc_h2_unsent_message_unittest.cpp
+++ b/test/brpc_h2_unsent_message_unittest.cpp
@@ -25,7 +25,7 @@
#include "butil/atomicops.h"
#include "brpc/policy/http_rpc_protocol.h"
#include "brpc/policy/http2_rpc_protocol.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
diff --git a/test/brpc_http_rpc_protocol_unittest.cpp
b/test/brpc_http_rpc_protocol_unittest.cpp
index cf23debd..ad631a2e 100644
--- a/test/brpc_http_rpc_protocol_unittest.cpp
+++ b/test/brpc_http_rpc_protocol_unittest.cpp
@@ -733,15 +733,19 @@ private:
butil::Status _destroying_st;
};
+#ifdef BUTIL_USE_ASAN
+static const int GENERAL_DELAY_US = 1000000; // 1s
+#else
static const int GENERAL_DELAY_US = 300000; // 0.3s
+#endif
TEST_F(HttpTest, read_long_body_progressively) {
+ DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
+ std::numeric_limits<size_t>::max());
butil::intrusive_ptr<ReadBody> reader;
{
const int port = 8923;
brpc::Server server;
- DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
- std::numeric_limits<size_t>::max());
EXPECT_EQ(0, server.AddService(&svc, brpc::SERVER_DOESNT_OWN_SERVICE));
EXPECT_EQ(0, server.Start(port, NULL));
{
@@ -823,12 +827,12 @@ TEST_F(HttpTest, read_short_body_progressively) {
}
TEST_F(HttpTest, read_progressively_after_cntl_destroys) {
+ DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
+ std::numeric_limits<size_t>::max());
butil::intrusive_ptr<ReadBody> reader;
{
const int port = 8923;
brpc::Server server;
- DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
- std::numeric_limits<size_t>::max());
EXPECT_EQ(0, server.AddService(&svc, brpc::SERVER_DOESNT_OWN_SERVICE));
EXPECT_EQ(0, server.Start(port, NULL));
{
@@ -870,11 +874,11 @@ TEST_F(HttpTest, read_progressively_after_cntl_destroys) {
TEST_F(HttpTest, read_progressively_after_long_delay) {
butil::intrusive_ptr<ReadBody> reader;
+ DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
+ std::numeric_limits<size_t>::max());
{
const int port = 8923;
brpc::Server server;
- DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
- std::numeric_limits<size_t>::max());
EXPECT_EQ(0, server.AddService(&svc, brpc::SERVER_DOESNT_OWN_SERVICE));
EXPECT_EQ(0, server.Start(port, NULL));
{
@@ -919,10 +923,10 @@ TEST_F(HttpTest, read_progressively_after_long_delay) {
}
TEST_F(HttpTest, skip_progressive_reading) {
- const int port = 8923;
- brpc::Server server;
DownloadServiceImpl svc(DONE_BEFORE_CREATE_PA,
std::numeric_limits<size_t>::max());
+ const int port = 8923;
+ brpc::Server server;
EXPECT_EQ(0, server.AddService(&svc, brpc::SERVER_DOESNT_OWN_SERVICE));
EXPECT_EQ(0, server.Start(port, NULL));
brpc::Channel channel;
@@ -1029,6 +1033,7 @@ TEST_F(HttpTest, broken_socket_stops_progressive_reading)
{
ASSERT_EQ(ECONNRESET, reader->destroying_status().error_code());
}
+#ifndef BUTIL_USE_ASAN
static const std::string TEST_PROGRESSIVE_HEADER = "Progressive";
static const std::string TEST_PROGRESSIVE_HEADER_VAL = "Progressive-val";
@@ -1144,6 +1149,8 @@ TEST_F(HttpTest,
server_end_read_short_body_progressively) {
ASSERT_FALSE(cntl.Failed());
}
+// Fixme!!! Server progressive reader has a heap-use-after-free bug detected
by ASan.
+// For details, see
https://github.com/apache/brpc/issues/2145#issuecomment-2329413363
TEST_F(HttpTest, server_end_read_failed) {
const int port = 8923;
brpc::ServiceOptions opt;
@@ -1180,6 +1187,7 @@ TEST_F(HttpTest, server_end_read_failed) {
channel.CallMethod(NULL, &cntl, NULL, NULL, NULL);
ASSERT_TRUE(cntl.Failed());
}
+#endif // BUTIL_USE_ASAN
TEST_F(HttpTest, http2_sanity) {
const int port = 8923;
diff --git a/test/brpc_hulu_pbrpc_protocol_unittest.cpp
b/test/brpc_hulu_pbrpc_protocol_unittest.cpp
index 7c971dc2..d00b9d45 100644
--- a/test/brpc_hulu_pbrpc_protocol_unittest.cpp
+++ b/test/brpc_hulu_pbrpc_protocol_unittest.cpp
@@ -27,7 +27,6 @@
#include <google/protobuf/descriptor.h>
#include "butil/time.h"
#include "butil/macros.h"
-#include "butil/gperftools_profiler.h"
#include "brpc/socket.h"
#include "brpc/acceptor.h"
#include "brpc/server.h"
diff --git a/test/brpc_input_messenger_unittest.cpp
b/test/brpc_input_messenger_unittest.cpp
index 00b14ed4..812c499a 100644
--- a/test/brpc_input_messenger_unittest.cpp
+++ b/test/brpc_input_messenger_unittest.cpp
@@ -21,9 +21,9 @@
#include <sys/types.h>
#include <sys/socket.h>
-#include <netdb.h> //
+#include <netdb.h>
#include <gtest/gtest.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/fd_utility.h"
@@ -180,7 +180,8 @@ TEST_F(MessengerTest, dispatch_tasks) {
}
sleep(1);
-
+
+
LOG(INFO) << "Begin to profile... (5 seconds)";
ProfilerStart("input_messenger.prof");
diff --git a/test/brpc_load_balancer_unittest.cpp
b/test/brpc_load_balancer_unittest.cpp
index 2abb869e..a0a62c27 100644
--- a/test/brpc_load_balancer_unittest.cpp
+++ b/test/brpc_load_balancer_unittest.cpp
@@ -23,7 +23,7 @@
#include <map>
#include <gtest/gtest.h>
#include "bthread/bthread.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/containers/doubly_buffered_data.h"
#include "brpc/describable.h"
#include "brpc/socket.h"
diff --git a/test/brpc_protobuf_json_unittest.cpp
b/test/brpc_protobuf_json_unittest.cpp
index 3565772e..5bb158d7 100644
--- a/test/brpc_protobuf_json_unittest.cpp
+++ b/test/brpc_protobuf_json_unittest.cpp
@@ -26,7 +26,7 @@
#include "butil/strings/string_util.h"
#include "butil/third_party/rapidjson/rapidjson.h"
#include "butil/time.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "json2pb/pb_to_json.h"
#include "json2pb/json_to_pb.h"
#include "json2pb/encode_decode.h"
@@ -505,8 +505,8 @@ TEST_F(ProtobufJsonTest, json_to_pb_perf_case) {
printf("----------test json to pb performance------------\n\n");
- std::string error;
-
+ std::string error;
+
ProfilerStart("json_to_pb_perf.prof");
butil::Timer timer;
bool res;
@@ -542,8 +542,8 @@ TEST_F(ProtobufJsonTest,
json_to_pb_encode_decode_perf_case) {
\"welcome\",
\"enum--type\":1},\"uid*\":\"welcome\"}], \
\"judge\":false, \"spur\":2, \"data:array\":[]}";
printf("----------test json to pb encode/decode
performance------------\n\n");
- std::string error;
-
+ std::string error;
+
ProfilerStart("json_to_pb_encode_decode_perf.prof");
butil::Timer timer;
bool res;
@@ -593,7 +593,7 @@ TEST_F(ProtobufJsonTest, json_to_pb_complex_perf_case) {
json2pb::Json2PbOptions options;
options.base64_to_bytes = false;
ProfilerStart("json_to_pb_complex_perf.prof");
- for (int i = 0; i < times; i++) {
+ for (int i = 0; i < times; i++) {
gss::message::gss_us_res_t data;
butil::IOBufAsZeroCopyInputStream stream(buf);
timer.start();
@@ -625,7 +625,7 @@ TEST_F(ProtobufJsonTest,
json_to_pb_to_string_complex_perf_case) {
json2pb::Json2PbOptions options;
options.base64_to_bytes = false;
ProfilerStart("json_to_pb_to_string_complex_perf.prof");
- for (int i = 0; i < times; i++) {
+ for (int i = 0; i < times; i++) {
gss::message::gss_us_res_t data;
timer.start();
res = json2pb::JsonToProtoMessage(info3, &data, options, &error);
@@ -1292,6 +1292,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_perf_case) {
printf("text:%s\n", text.data());
printf("----------test pb to json performance------------\n\n");
+
ProfilerStart("pb_to_json_perf.prof");
butil::Timer timer;
bool res;
@@ -1352,6 +1353,7 @@ TEST_F(ProtobufJsonTest,
pb_to_json_encode_decode_perf_case) {
printf("----------test pb to json encode decode
performance------------\n\n");
ProfilerStart("pb_to_json_encode_decode_perf.prof");
+
butil::Timer timer;
bool res;
float avg_time1 = 0;
@@ -1401,7 +1403,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
res = JsonToProtoMessage(info3, &data, option, &error);
ASSERT_TRUE(res) << error;
ProfilerStart("pb_to_json_complex_perf.prof");
- for (int i = 0; i < times; i++) {
+ for (int i = 0; i < times; i++) {
std::string error1;
timer.start();
butil::IOBuf buf;
@@ -1437,7 +1439,7 @@ TEST_F(ProtobufJsonTest,
pb_to_json_to_string_complex_perf_case) {
res = JsonToProtoMessage(info3, &data, option, &error);
ASSERT_TRUE(res);
ProfilerStart("pb_to_json_to_string_complex_perf.prof");
- for (int i = 0; i < times; i++) {
+ for (int i = 0; i < times; i++) {
std::string info4;
std::string error1;
timer.start();
diff --git a/test/brpc_snappy_compress_unittest.cpp
b/test/brpc_snappy_compress_unittest.cpp
index ee6a00e6..94b54dfd 100644
--- a/test/brpc_snappy_compress_unittest.cpp
+++ b/test/brpc_snappy_compress_unittest.cpp
@@ -20,7 +20,7 @@
// Date: 2015/01/20 19:01:06
#include <gtest/gtest.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/third_party/snappy/snappy.h"
#include "butil/macros.h"
#include "butil/iobuf.h"
diff --git a/test/brpc_socket_unittest.cpp b/test/brpc_socket_unittest.cpp
index 70dd8021..78b83503 100644
--- a/test/brpc_socket_unittest.cpp
+++ b/test/brpc_socket_unittest.cpp
@@ -24,7 +24,7 @@
#include <fcntl.h> // F_GETFD
#include <gtest/gtest.h>
#include <gflags/gflags.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/fd_utility.h"
diff --git a/test/bthread_cond_unittest.cpp b/test/bthread_cond_unittest.cpp
index 948b75de..035ff333 100644
--- a/test/bthread_cond_unittest.cpp
+++ b/test/bthread_cond_unittest.cpp
@@ -21,7 +21,7 @@
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/scoped_lock.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "bthread/bthread.h"
#include "bthread/condition_variable.h"
#include "bthread/stack.h"
diff --git a/test/bthread_dispatcher_unittest.cpp
b/test/bthread_dispatcher_unittest.cpp
index a2008020..669d9c4e 100644
--- a/test/bthread_dispatcher_unittest.cpp
+++ b/test/bthread_dispatcher_unittest.cpp
@@ -25,7 +25,7 @@
#include "butil/scoped_lock.h"
#include "butil/fd_utility.h"
#include "butil/logging.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "bthread/bthread.h"
#include "bthread/task_control.h"
#include "bthread/task_group.h"
@@ -192,6 +192,7 @@ void* client_thread(void* arg) {
}
}
}
+ free(buf);
return NULL;
}
@@ -256,7 +257,7 @@ TEST(DispatcherTest, dispatch_tasks) {
cm[i]->bytes = 0;
ASSERT_EQ(0, pthread_create(&cth[i], NULL, client_thread, cm[i]));
}
-
+
ProfilerStart("dispatcher.prof");
butil::Timer tm;
tm.start();
diff --git a/test/bthread_execution_queue_unittest.cpp
b/test/bthread_execution_queue_unittest.cpp
index 6ecad01b..1baaa777 100644
--- a/test/bthread_execution_queue_unittest.cpp
+++ b/test/bthread_execution_queue_unittest.cpp
@@ -22,7 +22,7 @@
#include <bthread/countdown_event.h>
#include "butil/time.h"
#include "butil/fast_rand.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
namespace {
bool stopped = false;
diff --git a/test/bthread_fd_unittest.cpp b/test/bthread_fd_unittest.cpp
index 49275c2c..91b89c21 100644
--- a/test/bthread_fd_unittest.cpp
+++ b/test/bthread_fd_unittest.cpp
@@ -16,13 +16,14 @@
// under the License.
#include "butil/compat.h"
+#include "butil/compiler_specific.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/utsname.h> // uname
#include <fcntl.h>
#include <gtest/gtest.h>
#include <pthread.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/fd_utility.h"
@@ -328,7 +329,7 @@ TEST(FDTest, ping_pong) {
}
tm.stop();
ProfilerStop();
- LOG(INFO) << "tid=" << REP*NCLIENT*1000000L/tm.u_elapsed();
+ LOG(INFO) << "tid=" << REP*NCLIENT * 1000000L / tm.u_elapsed();
stop = true;
for (size_t i = 0; i < NEPOLL; ++i) {
#if defined(OS_LINUX)
diff --git a/test/bthread_mutex_unittest.cpp b/test/bthread_mutex_unittest.cpp
index 3163876c..f839d063 100644
--- a/test/bthread_mutex_unittest.cpp
+++ b/test/bthread_mutex_unittest.cpp
@@ -25,7 +25,7 @@
#include "bthread/butex.h"
#include "bthread/task_control.h"
#include "bthread/mutex.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
namespace {
inline unsigned* get_butex(bthread_mutex_t & m) {
@@ -206,7 +206,7 @@ void PerfTest(Mutex* mutex,
}
g_started = true;
char prof_name[32];
- snprintf(prof_name, sizeof(prof_name), "mutex_perf_%d.prof",
++g_prof_name_counter);
+ snprintf(prof_name, sizeof(prof_name), "mutex_perf_%d.prof",
++g_prof_name_counter);
ProfilerStart(prof_name);
usleep(500 * 1000);
ProfilerStop();
diff --git a/test/bthread_rwlock_unittest.cpp b/test/bthread_rwlock_unittest.cpp
index 3318956b..815494ca 100644
--- a/test/bthread_rwlock_unittest.cpp
+++ b/test/bthread_rwlock_unittest.cpp
@@ -16,7 +16,7 @@
// under the License.
#include <gtest/gtest.h>
-#include <butil/gperftools_profiler.h>
+#include "gperftools_helper.h"
#include <bthread/rwlock.h>
namespace {
@@ -447,6 +447,7 @@ TEST(RWLockTest, pthread_rdlock_performance) {
long* res = NULL;
pthread_join(rth[i], (void**)&res);
printf("read thread %lu = %ldns\n", i, *res);
+ delete res;
}
pthread_join(wth, NULL);
#ifdef CHECK_RWLOCK
diff --git a/test/bthread_setconcurrency_unittest.cpp
b/test/bthread_setconcurrency_unittest.cpp
index 4d269b64..9e2e50d1 100644
--- a/test/bthread_setconcurrency_unittest.cpp
+++ b/test/bthread_setconcurrency_unittest.cpp
@@ -97,7 +97,7 @@ TEST(BthreadTest, setconcurrency_with_running_bthread) {
*odd = 0;
*even = 0;
std::vector<bthread_t> tids;
- const int N = 500;
+ const int N = 200;
for (int i = 0; i < N; ++i) {
bthread_t tid;
bthread_start_background(&tid, &BTHREAD_ATTR_SMALL, odd_thread, NULL);
@@ -155,7 +155,7 @@ TEST(BthreadTest, min_concurrency) {
ASSERT_EQ(1, set_min_concurrency(0)); // set min success
ASSERT_EQ(0, get_min_concurrency());
int conn = bthread_getconcurrency();
- int add_conn = 100;
+ int add_conn = 50;
ASSERT_EQ(0, set_min_concurrency(conn + 1)); // set min failed
ASSERT_EQ(0, get_min_concurrency());
diff --git a/test/bthread_unittest.cpp b/test/bthread_unittest.cpp
index 57f4fc82..1fbefb27 100644
--- a/test/bthread_unittest.cpp
+++ b/test/bthread_unittest.cpp
@@ -20,7 +20,7 @@
#include "butil/time.h"
#include "butil/macros.h"
#include "butil/logging.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "bthread/bthread.h"
#include "bthread/unstable.h"
#include "bthread/task_meta.h"
@@ -230,6 +230,7 @@ TEST_F(BthreadTest, backtrace) {
for (int i = 0; i < bt_cnt; ++i) {
puts(text[i]);
}
+ free(text);
}
void* show_self(void*) {
@@ -330,7 +331,7 @@ TEST_F(BthreadTest, small_threads) {
LOG(INFO) << "[Round " << j + 1 << "] bthread_start_urgent takes "
<< tm.n_elapsed()/N << "ns, sum=" << s;
ASSERT_EQ(N * (j + 1), (size_t)s);
-
+
// Check uniqueness of th
std::sort(th.begin(), th.end());
ASSERT_EQ(th.end(), std::unique(th.begin(), th.end()));
@@ -339,9 +340,14 @@ TEST_F(BthreadTest, small_threads) {
}
void* bthread_starter(void* void_counter) {
+ std::vector<bthread_t> ths;
while (!stop.load(butil::memory_order_relaxed)) {
bthread_t th;
EXPECT_EQ(0, bthread_start_urgent(&th, NULL, adding_func,
void_counter));
+ ths.push_back(th);
+ }
+ for (size_t i = 0; i < ths.size(); ++i) {
+ EXPECT_EQ(0, bthread_join(ths[i], NULL));
}
return NULL;
}
@@ -361,7 +367,7 @@ TEST_F(BthreadTest, start_bthreads_frequently) {
bthread_t th[con];
std::cout << "Perf with different parameters..." << std::endl;
- //ProfilerStart(prof_name);
+ ProfilerStart(prof_name);
for (int cur_con = 1; cur_con <= con; ++cur_con) {
stop = false;
for (int i = 0; i < cur_con; ++i) {
@@ -384,7 +390,7 @@ TEST_F(BthreadTest, start_bthreads_frequently) {
std::cout << sum << ",";
}
std::cout << std::endl;
- //ProfilerStop();
+ ProfilerStop();
delete [] counters;
}
diff --git a/test/bthread_work_stealing_queue_unittest.cpp
b/test/bthread_work_stealing_queue_unittest.cpp
index 82897533..92fbb910 100644
--- a/test/bthread_work_stealing_queue_unittest.cpp
+++ b/test/bthread_work_stealing_queue_unittest.cpp
@@ -109,6 +109,7 @@ TEST(WSQTest, sanity) {
for (size_t j = 0; j < res->size(); ++j, ++nstolen) {
values.push_back((*res)[j]);
}
+ delete res;
}
pthread_join(wth, NULL);
std::vector<value_type>* res = NULL;
@@ -116,6 +117,7 @@ TEST(WSQTest, sanity) {
for (size_t j = 0; j < res->size(); ++j, ++npopped) {
values.push_back((*res)[j]);
}
+ delete res;
value_type val;
while (q.pop(&val)) {
diff --git a/test/bvar_lock_timer_unittest.cpp
b/test/bvar_lock_timer_unittest.cpp
index 52b42f9b..f69ba3a2 100644
--- a/test/bvar_lock_timer_unittest.cpp
+++ b/test/bvar_lock_timer_unittest.cpp
@@ -22,7 +22,7 @@
#include <condition_variable>
#endif
#include <gtest/gtest.h>
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "bvar/utils/lock_timer.h"
namespace {
@@ -220,7 +220,7 @@ TEST_F(LockTimerTest, overhead) {
MutexWithLatencyRecorder<DummyMutex> m0(r0);
butil::Timer timer;
const size_t N = 1000 * 1000 * 10;
-
+
ProfilerStart("mutex_with_latency_recorder.prof");
timer.start();
for (size_t i = 0; i < N; ++i) {
diff --git a/test/gperftools_helper.h b/test/gperftools_helper.h
new file mode 100644
index 00000000..c507ade7
--- /dev/null
+++ b/test/gperftools_helper.h
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef TEST_GPERFTOOLS_HELPER_H_
+#define TEST_GPERFTOOLS_HELPER_H_
+
+#if defined(BRPC_ENABLE_CPU_PROFILER) || defined(BAIDU_RPC_ENABLE_CPU_PROFILER)
+#include "butil/gperftools_profiler.h"
+#else
+extern "C" {
+
+// If gperftools are not used, these functions are no-ops.
+inline int ProfilerStart(const char*) { return 0; }
+inline void ProfilerStop() {}
+inline void ProfilerFlush() {}
+
+} // extern "C"
+#endif // BRPC_ENABLE_CPU_PROFILER || BAIDU_RPC_ENABLE_CPU_PROFILER
+
+#endif // TEST_GPERFTOOLS_HELPER_H_
diff --git a/test/iobuf_unittest.cpp b/test/iobuf_unittest.cpp
index da36c180..b45a174c 100644
--- a/test/iobuf_unittest.cpp
+++ b/test/iobuf_unittest.cpp
@@ -291,6 +291,7 @@ TEST_F(IOBufTest, appendv) {
ASSERT_EQ(0, b.appendv(vec2, arraysize(vec2)));
ASSERT_EQ(full_len, b.size());
ASSERT_EQ(0, memcmp(str, b.to_string().data(), full_len));
+ free(str);
}
TEST_F(IOBufTest, reserve) {
@@ -345,6 +346,8 @@ TEST_F(IOBufTest, reserve) {
ASSERT_EQ("orang" + s2 + s1, b.to_string());
}
+#ifndef BUTIL_USE_ASAN
+// ASan will detect heap-buffer-overflow error casued by FakeBlock.
struct FakeBlock {
int nshared;
FakeBlock() : nshared(1) {}
@@ -451,6 +454,7 @@ TEST_F(IOBufTest, iobuf_as_queue) {
delete blocks[i];
}
}
+#endif // BUTIL_USE_ASAN
TEST_F(IOBufTest, iobuf_sanity) {
install_debug_allocator();
diff --git a/test/logging_unittest.cc b/test/logging_unittest.cc
index a431e01b..3a7576ff 100644
--- a/test/logging_unittest.cc
+++ b/test/logging_unittest.cc
@@ -4,7 +4,7 @@
#include "butil/basictypes.h"
#include "butil/logging.h"
-#include "butil/gperftools_profiler.h"
+#include "gperftools_helper.h"
#include "butil/files/temp_file.h"
#include "butil/popen.h"
#include <gtest/gtest.h>
@@ -540,6 +540,7 @@ TEST_F(LoggingTest, async_log) {
FLAGS_async_log = saved_async_log;
}
+#if defined(BRPC_ENABLE_CPU_PROFILER) || defined(BAIDU_RPC_ENABLE_CPU_PROFILER)
struct BAIDU_CACHELINE_ALIGNMENT PerfArgs {
const std::string* log;
int64_t counter;
@@ -643,6 +644,7 @@ TEST_F(LoggingTest, performance) {
FLAGS_async_log = saved_async_log;
}
+#endif // BRPC_ENABLE_CPU_PROFILER || BAIDU_RPC_ENABLE_CPU_PROFILER
} // namespace
diff --git a/test/mpsc_queue_unittest.cc b/test/mpsc_queue_unittest.cc
index f57a85f6..c651e3a0 100644
--- a/test/mpsc_queue_unittest.cc
+++ b/test/mpsc_queue_unittest.cc
@@ -2,9 +2,6 @@
#include <pthread.h>
#include "butil/containers/mpsc_queue.h"
-#define BAIDU_CLEAR_OBJECT_POOL_AFTER_ALL_THREADS_QUIT
-#include "butil/object_pool.h"
-
namespace {
const uint MAX_COUNT = 1000000;
diff --git a/test/popen_unittest.cpp b/test/popen_unittest.cpp
index efd307e4..81d4cdec 100644
--- a/test/popen_unittest.cpp
+++ b/test/popen_unittest.cpp
@@ -135,6 +135,7 @@ TEST(PopenTest, does_vfork_suspend_all_threads) {
<< " as=" << counter_after_sleep
<< std::endl;
ASSERT_EQ(cpid, waitpid(cpid, &ws, __WALL));
+ free(child_stack_mem);
}
#endif // OS_LINUX
diff --git a/test/run_tests.sh b/test/run_tests.sh
index 14297797..2e174777 100755
--- a/test/run_tests.sh
+++ b/test/run_tests.sh
@@ -26,7 +26,10 @@ test_bins="test_butil test_bvar bthread*unittest
brpc*unittest"
for test_bin in $test_bins; do
test_num=$((test_num + 1))
>&2 echo "[runtest] $test_bin"
- ./$test_bin
+ ASAN_OPTIONS="detect_leaks=0" ./$test_bin
+ # If ASan abort without detailed call stack of new/delete,
+ # try to disable fast_unwind_on_malloc, which would be a performance
killer.
+ # ASAN_OPTIONS="fast_unwind_on_malloc=0:detect_leaks=0" ./$test_bin
rc=$?
if [ $rc -ne 0 ]; then
failed_test="$test_bin"
diff --git a/test/security_unittest.cc b/test/security_unittest.cc
index d418ebbe..5121775d 100644
--- a/test/security_unittest.cc
+++ b/test/security_unittest.cc
@@ -22,6 +22,9 @@
#include <sys/mman.h>
#include <unistd.h>
#endif
+#include "butil/compiler_specific.h"
+
+#ifdef BUTIL_USE_ASAN
using std::nothrow;
using std::numeric_limits;
@@ -298,3 +301,5 @@ TEST(SecurityTest, TCMALLOC_TEST(RandomMemoryAllocations)) {
#endif // defined(OS_LINUX) && defined(__x86_64__)
} // namespace
+
+#endif // BUTIL_USE_ASAN
diff --git a/test/stack_trace_unittest.cc b/test/stack_trace_unittest.cc
index 992a7404..72b1b986 100644
--- a/test/stack_trace_unittest.cc
+++ b/test/stack_trace_unittest.cc
@@ -145,6 +145,7 @@ void CheckDebugOutputToStream(bool exclude_self) {
std::ostringstream os;
trace.OutputToStream(&os);
VLOG(1) << os.str();
+ free(addr);
}
// The test is used for manual testing, e.g., to see the raw output.
diff --git a/test/thread_key_unittest.cpp b/test/thread_key_unittest.cpp
index 5a95b8f0..758a6791 100644
--- a/test/thread_key_unittest.cpp
+++ b/test/thread_key_unittest.cpp
@@ -309,8 +309,6 @@ TEST(ThreadLocalTest, thread_key_multi_thread) {
}
}
-DEFINE_bool(test_pthread_key, true, "test pthread_key");
-
struct BAIDU_CACHELINE_ALIGNMENT ThreadKeyPerfArgs {
pthread_key_t pthread_key;
ThreadKey* thread_key;
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@brpc.apache.org
For additional commands, e-mail: dev-h...@brpc.apache.org
