372046933 opened a new issue, #1995:
URL: https://github.com/apache/incubator-brpc/issues/1995
**Describe the bug (描述bug)**
`IOBuf` cannot `append_user_data` with address starts after
`RegisterMemoryForRdma`
```bash
W20221115 18:15:32.455838 79345 rdma_endpoint.cpp:727] Memory not registered
for rdma. Is this iobuf allocated before calling GlobalRdmaInitializeOrDie? Or
just forget to call RegisterMemoryForRdma for your own buffer?
W20221115 18:15:32.455842 79345 socket.cpp:1705] Fail to keep-write into
Socket{id=2 fd=1034 addr=0.0.0.0:8002:53496} (0x55b33af574d0): Unknown error
3002 [3002]
E20221115 18:15:32.455933 79247 server2.cpp:95] RPC call failed: [E3002]Fail
to keep-write into Socket{id=2 fd=1034 addr=0.0.0.0:8002:53496}
(0x0x55b33af574d0): Memory not registered for RDMA
```
**To Reproduce (复现方法)**
```c++
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gflags/gflags.h>
#include "butil/atomicops.h"
#include "butil/logging.h"
#include "butil/time.h"
#include "brpc/channel.h"
#include "brpc/rdma/rdma_helper.h"
#include "brpc/server.h"
#include "bvar/variable.h"
#include "test.pb.h"
DEFINE_int32(port, 8002, "TCP Port of this server");
DEFINE_bool(use_rdma, true, "Use RDMA or not");
namespace brpc{
namespace rdma {
DECLARE_bool(rdma_trace_verbose);
}
} // namespace brpc
butil::atomic<uint64_t> g_last_time(0);
namespace test {
class PerfTestServiceImpl : public PerfTestService {
public:
PerfTestServiceImpl() {}
~PerfTestServiceImpl() {};
void Test(google::protobuf::RpcController* cntl_base,
const PerfTestRequest* request,
PerfTestResponse* response,
google::protobuf::Closure* done) {
brpc::ClosureGuard done_guard(done);
uint64_t last = g_last_time.load(butil::memory_order_relaxed);
uint64_t now = butil::monotonic_time_us();
if (now > last && now - last > 100000) {
if (g_last_time.exchange(now, butil::memory_order_relaxed) ==
last) {
response->set_cpu_usage(bvar::Variable::describe_exposed("process_cpu_usage"));
} else {
response->set_cpu_usage("");
}
} else {
response->set_cpu_usage("");
}
brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base);
if (request->echo_attachment()) {
cntl->response_attachment().append(cntl->request_attachment());
}
if (!cntl->request_attachment().empty()) {
std::string tmp;
cntl->request_attachment().cutn(&tmp, 5);
LOG(INFO) << tmp;
}
}
};
}
void run_client() {
brpc::rdma::GlobalRdmaInitializeOrDie();
brpc::ChannelOptions options;
options.use_rdma = FLAGS_use_rdma;
options.protocol = "baidu_std";
options.connection_type = "single";
options.timeout_ms = 10000;
options.max_retry = 0;
std::string server = "0.0.0.0:8002";
brpc::Channel* _channel = new brpc::Channel();
if (_channel->Init(server.c_str(), &options) != 0) {
LOG(ERROR) << "Fail to initialize channel";
return;
}
brpc::Controller cntl;
test::PerfTestResponse response;
test::PerfTestRequest request;
char buffer[16] = {'a', 'b', 'c'};
brpc::rdma::RegisterMemoryForRdma(buffer, 16);
cntl.request_attachment().append_user_data(buffer + 1, 3, [](void*) {});
request.set_echo_attachment(false);
test::PerfTestService_Stub stub(_channel);
stub.Test(&cntl, &request, &response, NULL);
if (cntl.Failed()) {
LOG(ERROR) << "RPC call failed: " << cntl.ErrorText();
} else {
LOG(INFO) << "RPC call success";
}
}
int main(int argc, char* argv[]) {
brpc::rdma::fLB::FLAGS_rdma_trace_verbose = true;
GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_use_rdma) {
LOG(INFO) << "Call GlobalRdmaInitializeOrDie";
brpc::rdma::GlobalRdmaInitializeOrDie();
}
brpc::Server server;
test::PerfTestServiceImpl perf_test_service_impl;
if (server.AddService(&perf_test_service_impl,
brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
LOG(ERROR) << "Fail to add service";
return -1;
}
g_last_time.store(0, butil::memory_order_relaxed);
brpc::ServerOptions options;
options.use_rdma = FLAGS_use_rdma;
if (server.Start(FLAGS_port, &options) != 0) {
LOG(ERROR) << "Fail to start EchoServer";
return -1;
}
run_client();
server.RunUntilAskedToQuit();
return 0;
}
```
**Expected behavior (期望行为)**
`IOBuf` can `append_user_data` with data in `RegisterMemoryForRdma`
**Versions (各种版本)**
OS: Ubuntu 22.04
Compiler: GCC 11.2.0
brpc: 3d90221107000133c2d7f3ccc6372680fb6af27d
protobuf:
**Additional context/screenshots (更多上下文/截图)**
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
