This is an automated email from the ASF dual-hosted git repository.
chenBright pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git
The following commit(s) were added to refs/heads/master by this push:
new 886f9fdae Reject over-limit requests on the nshead server instead of
CHECK (#3723)
886f9fdae is described below
commit 886f9fdae726cc761ac5353580e015f9bbae24e0
Author: Weibing Wang <[email protected]>
AuthorDate: Mon Nov 9 23:32:36 2026 +0800
Reject over-limit requests on the nshead server instead of CHECK (#3723)
ProcessNsheadRequest() wrapped MethodStatus::OnRequested() in CHECK(),
treating a rejection of the concurrency limiter as an impossible state.
Both built-in limiters really return false under overload, so the first
request that reaches the limit trips the CHECK: with -crash_on_fatal_log
the process aborts, and with the default configuration a FATAL log with
a stack trace is emitted while the rejected request is served anyway,
which silently disables the limit configured by the operator.
Handle the rejection like the other protocols do: fail the controller
with ELIMIT and let the handler (and the pb adaptor) answer the client.
Note that OnRequested() counts the request even when it is rejected, and
ConcurrencyRemover in NsheadClosure::Run() counts it out, so the
concurrency accounting stays balanced.
The same rejection was discarded in NsheadPbServiceAdaptor, which is
used by nshead-based pb protocols (public_pbrpc, nova_pbrpc,
nshead_mcpack): the method-level limiter was never enforced there.
Fail with ELIMIT as well.
Co-authored-by: brpc-oncall <brpc-oncall@localhost>
---
src/brpc/nshead_pb_service_adaptor.cpp | 7 +-
src/brpc/policy/nshead_protocol.cpp | 16 ++-
test/brpc_server_unittest.cpp | 171 +++++++++++++++++++++++++++++++++
3 files changed, 189 insertions(+), 5 deletions(-)
diff --git a/src/brpc/nshead_pb_service_adaptor.cpp
b/src/brpc/nshead_pb_service_adaptor.cpp
index 3be93224e..e87e7d60c 100644
--- a/src/brpc/nshead_pb_service_adaptor.cpp
+++ b/src/brpc/nshead_pb_service_adaptor.cpp
@@ -127,7 +127,12 @@ void NsheadPbServiceAdaptor::ProcessNsheadRequest(
break;
}
pbdone->status = mp->status;
- mp->status->OnRequested();
+ if (!mp->status->OnRequested()) {
+ controller->SetFailed(ELIMIT, "Reached %s's max_concurrency=%d",
+ meta->full_method_name().c_str(),
+ mp->status->MaxConcurrency());
+ break;
+ }
google::protobuf::Service* svc = mp->service;
const google::protobuf::MethodDescriptor* method = mp->method;
diff --git a/src/brpc/policy/nshead_protocol.cpp
b/src/brpc/policy/nshead_protocol.cpp
index 5042b9221..88e46f9a9 100644
--- a/src/brpc/policy/nshead_protocol.cpp
+++ b/src/brpc/policy/nshead_protocol.cpp
@@ -257,10 +257,7 @@ void ProcessNsheadRequest(InputMessageBase* msg_base) {
// Switch to service-specific error.
non_service_error.release();
MethodStatus* method_status = service->_status;
- if (method_status) {
- CHECK(method_status->OnRequested());
- }
-
+
void* sub_space = nullptr;
if (service->_additional_space) {
sub_space = (char*)space + sizeof(NsheadClosure);
@@ -310,6 +307,17 @@ void ProcessNsheadRequest(InputMessageBase* msg_base) {
}
do {
+ // The limiter rejects the request when the concurrency reaches the
+ // upper limit, which is a normal situation under overload instead
+ // of an impossible state. Note that OnRequested() counts the
+ // request in either case, and ConcurrencyRemover in
+ // NsheadClosure::Run() counts it out.
+ if (method_status && !method_status->OnRequested()) {
+ cntl->SetFailed(ELIMIT,
+ "Reached NsheadService's max_concurrency=%d",
+ method_status->MaxConcurrency());
+ break;
+ }
if (!server->IsRunning()) {
cntl->SetFailed(ELOGOFF, "Server is stopping");
break;
diff --git a/test/brpc_server_unittest.cpp b/test/brpc_server_unittest.cpp
index 807320869..76bf79bb1 100644
--- a/test/brpc_server_unittest.cpp
+++ b/test/brpc_server_unittest.cpp
@@ -60,6 +60,7 @@
#include "brpc/socket_map.h"
#include "brpc/controller.h"
#include "brpc/compress.h"
+#include "brpc/policy/public_pbrpc_protocol.h"
#include "echo.pb.h"
#include "v1.pb.h"
#include "v2.pb.h"
@@ -2361,6 +2362,176 @@ TEST_F(ServerTest,
nshead_service_is_not_served_on_internal_port) {
ASSERT_EQ(0, server.Join());
}
+// A request whose body is BLOCK_REQUEST holds the concurrency slot until the
+// test signals _release, so that the limit can be reached deterministically.
+static const char* const BLOCK_REQUEST = "block";
+
+class BlockingNsheadService : public brpc::NsheadService {
+public:
+ void ProcessNsheadRequest(const brpc::Server&,
+ brpc::Controller* cntl,
+ const brpc::NsheadMessage& request,
+ brpc::NsheadMessage* response,
+ brpc::NsheadClosure* done) override {
+ brpc::ClosureGuard done_guard(done);
+ if (cntl->Failed()) {
+ // nshead carries no error field, echo the code back.
+ response->body.append(butil::string_printf("%d",
cntl->ErrorCode()));
+ return;
+ }
+ if (request.body.equals(BLOCK_REQUEST)) {
+ _entered.signal();
+ _release.wait();
+ }
+ response->body.append(request.body);
+ }
+
+ bthread::CountdownEvent _entered{1};
+ bthread::CountdownEvent _release{1};
+};
+
+// A request rejected by the concurrency limiter of the NsheadService is a
+// normal situation instead of an impossible one: it must be failed with
+// ELIMIT and reach the user handler (serving it anyway silently disables the
+// limit configured by the operator), and the rejected concurrency must be
+// counted out so that later requests are not rejected forever.
+TEST_F(ServerTest, nshead_service_rejects_request_on_concurrency_limit) {
+ brpc::Server server;
+ brpc::ServerOptions opt;
+ BlockingNsheadService* nshead_service = new BlockingNsheadService;
+ opt.nshead_service = nshead_service;
+ opt.method_max_concurrency = "1";
+ ASSERT_EQ(0, server.Start("127.0.0.1:0", &opt));
+
+ brpc::ChannelOptions copt;
+ copt.protocol = brpc::PROTOCOL_NSHEAD;
+ copt.connection_type = brpc::CONNECTION_TYPE_POOLED;
+ copt.max_retry = 0;
+ brpc::Channel chan;
+ ASSERT_EQ(0, chan.Init(server.listen_address(), &copt));
+
+ // Take the only slot and hold it.
+ brpc::NsheadMessage blocking_req;
+ brpc::NsheadMessage blocking_res;
+ brpc::Controller blocking_cntl;
+ blocking_req.body.append(BLOCK_REQUEST);
+ chan.CallMethod(nullptr, &blocking_cntl, &blocking_req,
+ &blocking_res, brpc::DoNothing());
+ ASSERT_EQ(0, nshead_service->_entered.wait());
+
+ brpc::NsheadMessage req;
+ brpc::NsheadMessage res;
+ brpc::Controller cntl;
+ req.body.append(EXP_REQUEST);
+ chan.CallMethod(nullptr, &cntl, &req, &res, nullptr);
+
+ // Release the slot before checking the result, otherwise failures below
+ // would leave the blocking request hanging in the server.
+ nshead_service->_release.signal();
+ brpc::Join(blocking_cntl.call_id());
+ ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
+ EXPECT_EQ(butil::string_printf("%d", brpc::ELIMIT), res.body.to_string());
+ ASSERT_FALSE(blocking_cntl.Failed()) << blocking_cntl.ErrorText();
+ ASSERT_EQ(BLOCK_REQUEST, blocking_res.body.to_string());
+
+ bool accepted = false;
+ for (int i = 0; i < 100 && !accepted; ++i) {
+ cntl.Reset();
+ res.body.clear();
+ chan.CallMethod(nullptr, &cntl, &req, &res, nullptr);
+ ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
+ accepted = res.body.equals(EXP_REQUEST);
+ if (!accepted) {
+ bthread_usleep(10000);
+ }
+ }
+ ASSERT_TRUE(accepted) << res.body.to_string();
+
+ ASSERT_EQ(0, server.Stop(0));
+ ASSERT_EQ(0, server.Join());
+}
+
+// A pb method served through the nshead adaptor is protected by its own
+// concurrency limiter, the rejection must be taken as well.
+class BlockingEchoService : public test::EchoService {
+public:
+ void Echo(google::protobuf::RpcController*,
+ const test::EchoRequest* request,
+ test::EchoResponse* response,
+ google::protobuf::Closure* done) override {
+ brpc::ClosureGuard done_guard(done);
+ if (request->message() == BLOCK_REQUEST) {
+ _entered.signal();
+ _release.wait();
+ }
+ response->set_message(request->message());
+ }
+
+ bthread::CountdownEvent _entered{1};
+ bthread::CountdownEvent _release{1};
+};
+
+// Same as nshead_service_rejects_request_on_concurrency_limit, for a pb
service
+// serving through the nshead adaptor: the limit is checked against the method.
+TEST_F(ServerTest, nshead_adaptor_rejects_request_on_method_concurrency_limit)
{
+ brpc::Server server;
+ BlockingEchoService service;
+ ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE));
+ brpc::ServerOptions opt;
+ opt.nshead_service = new brpc::policy::PublicPbrpcServiceAdaptor;
+ opt.method_max_concurrency = "1";
+ ASSERT_EQ(0, server.Start("127.0.0.1:0", &opt));
+
+ brpc::ChannelOptions copt;
+ copt.protocol = "public_pbrpc";
+ copt.connection_type = brpc::CONNECTION_TYPE_POOLED;
+ copt.max_retry = 0;
+ brpc::Channel chan;
+ ASSERT_EQ(0, chan.Init(server.listen_address(), &copt));
+ test::EchoService_Stub stub(&chan);
+
+ // Take the only slot and hold it.
+ test::EchoRequest blocking_req;
+ test::EchoResponse blocking_res;
+ brpc::Controller blocking_cntl;
+ blocking_req.set_message(BLOCK_REQUEST);
+ stub.Echo(&blocking_cntl, &blocking_req, &blocking_res, brpc::DoNothing());
+ ASSERT_EQ(0, service._entered.wait());
+
+ test::EchoRequest req;
+ test::EchoResponse res;
+ brpc::Controller cntl;
+ req.set_message(EXP_REQUEST);
+ stub.Echo(&cntl, &req, &res, nullptr);
+
+ // Release the slot before checking the result, otherwise failures below
+ // would leave the blocking request hanging in the server.
+ service._release.signal();
+ brpc::Join(blocking_cntl.call_id());
+ ASSERT_EQ(brpc::ELIMIT, cntl.ErrorCode()) << cntl.ErrorText();
+ ASSERT_FALSE(blocking_cntl.Failed()) << blocking_cntl.ErrorText();
+ ASSERT_EQ(BLOCK_REQUEST, blocking_res.message());
+
+ // The slot is released, requests are no longer rejected. The response
+ // reaches the client before the pb method recycles the concurrency, hence
+ // the retry.
+ bool accepted = false;
+ for (int i = 0; i < 100 && !accepted; ++i) {
+ cntl.Reset();
+ res.Clear();
+ stub.Echo(&cntl, &req, &res, nullptr);
+ accepted = !cntl.Failed();
+ if (!accepted) {
+ bthread_usleep(10000);
+ }
+ }
+ ASSERT_TRUE(accepted) << cntl.ErrorText();
+ ASSERT_EQ(EXP_REQUEST, res.message());
+
+ ASSERT_EQ(0, server.Stop(0));
+ ASSERT_EQ(0, server.Join());
+}
+
// A service-name-only URL is dispatched to the builtin BadMethodService which
// lists the methods of the service.
void CallServiceWithoutMethodByHttp(const butil::EndPoint& ep,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]