acelyc111 commented on code in PR #1583: URL: https://github.com/apache/incubator-pegasus/pull/1583#discussion_r1338018553
########## src/http/test/http_client_test.cpp: ########## @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <fmt/core.h> +#include <gtest/gtest.h> +// IWYU pragma: no_include <gtest/gtest-message.h> +// IWYU pragma: no_include <gtest/gtest-param-test.h> +// IWYU pragma: no_include <gtest/gtest-test-part.h> +#include <cstring> +#include <iostream> +#include <string> +#include <vector> + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; Review Comment: Is the output for debuging? If you want to check what the actual value is when test failed, you can use the macro like this: ``` ASSERT_*(x,x) << actual_description; ``` ########## thirdparty/CMakeLists.txt: ########## @@ -301,6 +297,20 @@ ExternalProject_Add(curl --without-libssh2 --without-ssl --without-libidn + --without-zstd) +if (APPLE) + set(CURL_OPTIONS + ${CURL_OPTIONS} + --without-nghttp2 + --without-libidn2 + --without-brotli) +endif () +ExternalProject_Add(curl + URL ${OSS_URL_PREFIX}/curl-8.3.0.tar.gz Review Comment: How about submit the upgrading of the thirdparty in another separeted pull request? ########## src/http/http_client.cpp: ########## @@ -0,0 +1,336 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/http_client.h" + +#include <fmt/core.h> +#include <limits> +#include <utility> + +#include "curl/curl.h" +#include "utils/error_code.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +DSN_DEFINE_uint32(http, + curl_timeout_ms, + 10000, + "The maximum time in milliseconds that you allow the libcurl transfer operation " + "to complete"); + +http_client::http_client() + : _curl(nullptr), + _method(http_method::GET), + _recv_callback(nullptr), + _header_changed(true), + _header_list(nullptr) +{ + // Since `kErrorBufferBytes` is private, `static_assert` have to be put in constructor. + static_assert(http_client::kErrorBufferBytes >= CURL_ERROR_SIZE, + "The error buffer used by libcurl must be at least CURL_ERROR_SIZE bytes big"); + + clear_error_buf(); +} + +http_client::~http_client() +{ + if (_curl != nullptr) { + curl_easy_cleanup(_curl); + _curl = nullptr; + } + + free_header_list(); +} + +namespace { + +inline dsn::error_code to_error_code(CURLcode code) +{ + if (code == CURLE_OK) { Review Comment: How about using `switch..case..`? ########## src/utils/error_code.h: ########## @@ -180,6 +180,10 @@ DEFINE_ERR_CODE(ERR_RANGER_POLICIES_NO_NEED_UPDATE) DEFINE_ERR_CODE(ERR_RDB_CORRUPTION) DEFINE_ERR_CODE(ERR_DISK_IO_ERROR) + +DEFINE_ERR_CODE(ERR_CURL_CONNECT_FAILED) +DEFINE_ERR_CODE(ERR_CURL_WRITE_ERROR) +DEFINE_ERR_CODE(ERR_CURL_FAILED) Review Comment: I'm a bit of confuesd of the relationship between `ERR_CURL_FAILED` and other `ERR_CURL_*_FAILED/ERROR` according to the names, what's the relationship of them? ########## src/http/test/http_client_test.cpp: ########## @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <fmt/core.h> +#include <gtest/gtest.h> +// IWYU pragma: no_include <gtest/gtest-message.h> +// IWYU pragma: no_include <gtest/gtest-param-test.h> +// IWYU pragma: no_include <gtest/gtest-test-part.h> +#include <cstring> +#include <iostream> +#include <string> +#include <vector> + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get";)); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_CONNECT_FAILED, err.code()); + + std::cout << "failed to connect: "; Review Comment: It it for debuging? ########## src/http/test/http_client_test.cpp: ########## @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <fmt/core.h> +#include <gtest/gtest.h> +// IWYU pragma: no_include <gtest/gtest-message.h> +// IWYU pragma: no_include <gtest/gtest-param-test.h> +// IWYU pragma: no_include <gtest/gtest-test-part.h> +#include <cstring> +#include <iostream> +#include <string> +#include <vector> + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get";)); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_CONNECT_FAILED, err.code()); + + std::cout << "failed to connect: "; + + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed to connect to 127.0.0.1 port 20000: Connection refused" + // * msg="Failed to connect to 127.0.0.1 port 20000 after 0 ms: Connection refused" + // Thus we don't check if `msg` fields are consistent. + const std::string expected_description_prefix( + "ERR_CURL_CONNECT_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20000/test/get): code=7, " + "desc=\"Couldn't connect to server\""); + check_expected_description_prefix(expected_description_prefix, err); +} + +TEST(HttpClientTest, Callback) +{ + http_client client; + ASSERT_TRUE(client.init()); + + ASSERT_TRUE(client.set_url("http://127.0.0.1:20001/test/get";)); + ASSERT_TRUE(client.with_get_method()); + + auto callback = [](const void *, size_t) { return false; }; + + const auto &err = client.exec_method(callback); + ASSERT_EQ(dsn::ERR_CURL_WRITE_ERROR, err.code()); + + long actual_http_status; + ASSERT_TRUE(client.get_http_status(actual_http_status)); + EXPECT_EQ(200, actual_http_status); + + std::cout << "failed for callback: "; Review Comment: It it for debuging? ########## src/http/test/http_client_test.cpp: ########## @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <fmt/core.h> +#include <gtest/gtest.h> +// IWYU pragma: no_include <gtest/gtest-message.h> +// IWYU pragma: no_include <gtest/gtest-param-test.h> +// IWYU pragma: no_include <gtest/gtest-test-part.h> +#include <cstring> +#include <iostream> +#include <string> +#include <vector> + +#include "http/http_client.h" +#include "http/http_method.h" +#include "utils/error_code.h" +#include "utils/errors.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +void check_expected_description_prefix(const std::string &expected_description_prefix, + const dsn::error_s &err) +{ + const std::string actual_description(err.description()); + std::cout << actual_description << std::endl; + + ASSERT_LT(expected_description_prefix.size(), actual_description.size()); + EXPECT_EQ(expected_description_prefix, + actual_description.substr(0, expected_description_prefix.size())); +} + +TEST(HttpClientTest, Connect) +{ + http_client client; + ASSERT_TRUE(client.init()); + + // No one has listened on port 20000, thus this would lead to "Connection refused". + ASSERT_TRUE(client.set_url("http://127.0.0.1:20000/test/get";)); + + const auto &err = client.exec_method(); + ASSERT_EQ(dsn::ERR_CURL_CONNECT_FAILED, err.code()); + + std::cout << "failed to connect: "; + + // We just check the prefix of description, including `method`, `url`, `code` and `desc`. + // The `msg` differ in various systems, such as: + // * msg="Failed to connect to 127.0.0.1 port 20000: Connection refused" + // * msg="Failed to connect to 127.0.0.1 port 20000 after 0 ms: Connection refused" + // Thus we don't check if `msg` fields are consistent. + const std::string expected_description_prefix( + "ERR_CURL_CONNECT_FAILED: failed to perform http request(" + "method=GET, url=http://127.0.0.1:20000/test/get): code=7, " + "desc=\"Couldn't connect to server\""); + check_expected_description_prefix(expected_description_prefix, err); Review Comment: It would be better to wrap the function call with `NO_FATALS()` which is > Macros that execute statement and check that it doesn't generate new fatal failures in the current thread. ########## src/http/http_client.cpp: ########## @@ -0,0 +1,336 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/http_client.h" + +#include <fmt/core.h> +#include <limits> +#include <utility> + +#include "curl/curl.h" +#include "utils/error_code.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" + +namespace dsn { + +DSN_DEFINE_uint32(http, + curl_timeout_ms, + 10000, + "The maximum time in milliseconds that you allow the libcurl transfer operation " + "to complete"); + +http_client::http_client() + : _curl(nullptr), + _method(http_method::GET), + _recv_callback(nullptr), + _header_changed(true), + _header_list(nullptr) +{ + // Since `kErrorBufferBytes` is private, `static_assert` have to be put in constructor. + static_assert(http_client::kErrorBufferBytes >= CURL_ERROR_SIZE, + "The error buffer used by libcurl must be at least CURL_ERROR_SIZE bytes big"); + + clear_error_buf(); +} + +http_client::~http_client() +{ + if (_curl != nullptr) { + curl_easy_cleanup(_curl); + _curl = nullptr; + } + + free_header_list(); +} + +namespace { + +inline dsn::error_code to_error_code(CURLcode code) +{ + if (code == CURLE_OK) { + return dsn::ERR_OK; + } + + if (code == CURLE_COULDNT_CONNECT) { + return dsn::ERR_CURL_CONNECT_FAILED; + } + + if (code == CURLE_OPERATION_TIMEDOUT) { + return dsn::ERR_TIMEOUT; + } + + if (code == CURLE_WRITE_ERROR) { + return dsn::ERR_CURL_WRITE_ERROR; + } + + return dsn::ERR_CURL_FAILED; +} + +} // anonymous namespace + +#define RETURN_IF_CURL_NOT_OK(expr, ...) \ + do { \ + const auto code = (expr); \ + if (dsn_unlikely(code != CURLE_OK)) { \ + std::string msg(fmt::format("{}: {}", fmt::format(__VA_ARGS__), to_error_msg(code))); \ + return dsn::error_s::make(to_error_code(code), msg); \ + } \ + } while (0) + +#define RETURN_IF_SETOPT_NOT_OK(opt, input) \ + RETURN_IF_CURL_NOT_OK(curl_easy_setopt(_curl, opt, input), \ + "failed to set " #opt " to" \ + " " #input) + +#define RETURN_IF_GETINFO_NOT_OK(info, output) \ + RETURN_IF_CURL_NOT_OK(curl_easy_getinfo(_curl, info, output), "failed to get from " #info) + +#define RETURN_IF_EXEC_METHOD_NOT_OK() \ + RETURN_IF_CURL_NOT_OK(curl_easy_perform(_curl), \ + "failed to perform http request(method={}, url={})", \ + enum_to_string(_method), \ + _url) + +dsn::error_s http_client::init() +{ + if (_curl == nullptr) { + _curl = curl_easy_init(); + if (_curl == nullptr) { + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "fail to initialize curl"); + } + } else { + curl_easy_reset(_curl); + } + + clear_header_fields(); + free_header_list(); + + // Additional messages for errors are needed. + clear_error_buf(); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_ERRORBUFFER, _error_buf); + + // Set with NOSIGNAL since we are multi-threaded. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_NOSIGNAL, 1L); + + // Redirects are supported. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_FOLLOWLOCATION, 1L); + + // Before 8.3.0, CURLOPT_MAXREDIRS was unlimited. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_MAXREDIRS, 20); + + // Set common timeout for transfer operation. Users could also change it with their + // custom values by `set_timeout`. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, static_cast<long>(FLAGS_curl_timeout_ms)); + + // A lambda can only be converted to a function pointer if it does not capture: + // https://stackoverflow.com/questions/28746744/passing-capturing-lambda-as-function-pointer + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf + curl_write_callback callback = [](char *buffer, size_t size, size_t nmemb, void *param) { + http_client *client = reinterpret_cast<http_client *>(param); + return client->on_response_data(buffer, size * nmemb); + }; + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEFUNCTION, callback); + + // This http_client object itself is passed to the callback function. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_WRITEDATA, reinterpret_cast<void *>(this)); + + return dsn::error_s::ok(); +} + +void http_client::clear_error_buf() { _error_buf[0] = 0; } + +bool http_client::is_error_buf_empty() const { return _error_buf[0] == 0; } + +std::string http_client::to_error_msg(CURLcode code) const +{ + std::string err_msg = + fmt::format("code={}, desc=\"{}\"", static_cast<int>(code), curl_easy_strerror(code)); + if (is_error_buf_empty()) { + return err_msg; + } + + err_msg += fmt::format(", msg=\"{}\"", _error_buf); + return err_msg; +} + +// `data` passed to this function is NOT null-terminated. +// `length` might be zero. +size_t http_client::on_response_data(const void *data, size_t length) +{ + if (_recv_callback == nullptr) { + return length; + } + + if (!(*_recv_callback)) { + // callback function is empty. + return length; + } + + // According to libcurl, callback should return the number of bytes actually taken care of. + // If that amount differs from the amount passed to callback function, it would signals an + // error condition. This causes the transfer to get aborted and the libcurl function used + // returns CURLE_WRITE_ERROR. Therefore, here we just return the max limit of size_t for + // failure. + // + // See https://curl.se/libcurl/c/CURLOPT_WRITEFUNCTION.html for details. + return (*_recv_callback)(data, length) ? length : std::numeric_limits<size_t>::max(); +} + +dsn::error_s http_client::set_url(const std::string &url) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_URL, url.c_str()); + + _url = url; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_post_method(const std::string &data) +{ + // No need to enable CURLOPT_POST by `RETURN_IF_SETOPT_NOT_OK(CURLOPT_POST, 1L)`, since using + // either of CURLOPT_POSTFIELDS or CURLOPT_COPYPOSTFIELDS implies setting CURLOPT_POST to 1. + // See https://curl.se/libcurl/c/CURLOPT_POSTFIELDS.html for details. + RETURN_IF_SETOPT_NOT_OK(CURLOPT_POSTFIELDSIZE, static_cast<long>(data.size())); + RETURN_IF_SETOPT_NOT_OK(CURLOPT_COPYPOSTFIELDS, data.data()); + _method = http_method::POST; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::with_get_method() { return set_method(http_method::GET); } + +dsn::error_s http_client::set_method(http_method method) +{ + // No need to process the case of http_method::POST, since it should be enabled by + // `with_post_method`. + switch (method) { + case http_method::GET: + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPGET, 1L); + break; + default: + LOG_FATAL("Unsupported http_method"); + } + + _method = method; + return dsn::error_s::ok(); +} + +dsn::error_s http_client::set_timeout(long timeout_ms) +{ + RETURN_IF_SETOPT_NOT_OK(CURLOPT_TIMEOUT_MS, timeout_ms); + return dsn::error_s::ok(); +} + +void http_client::clear_header_fields() +{ + _header_fields.clear(); + + _header_changed = true; +} + +void http_client::free_header_list() +{ + if (_header_list == nullptr) { + return; + } + + curl_slist_free_all(_header_list); + _header_list = nullptr; +} + +void http_client::set_header_field(dsn::string_view key, dsn::string_view val) +{ + _header_fields[std::string(key)] = std::string(val); + _header_changed = true; +} + +void http_client::set_accept(dsn::string_view val) { set_header_field("Accept", val); } + +void http_client::set_content_type(dsn::string_view val) { set_header_field("Content-Type", val); } + +dsn::error_s http_client::process_header() +{ + if (!_header_changed) { + return dsn::error_s::ok(); + } + + free_header_list(); + + for (const auto &field : _header_fields) { + auto str = fmt::format("{}: {}", field.first, field.second); + + // A null pointer is returned if anything went wrong, otherwise the new list pointer is + // returned. To avoid overwriting an existing non-empty list on failure, the new list + // should be returned to a temporary variable which can be tested for NULL before updating + // the original list pointer. (https://curl.se/libcurl/c/curl_slist_append.html) + struct curl_slist *temp = curl_slist_append(_header_list, str.c_str()); + if (temp == nullptr) { + free_header_list(); + return dsn::error_s::make(dsn::ERR_CURL_FAILED, "curl_slist_append failed"); + } + _header_list = temp; + } + + // This would work well even if `_header_list` is NULL pointer. Pass a NULL to this option + // to reset back to no custom headers. (https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) + RETURN_IF_SETOPT_NOT_OK(CURLOPT_HTTPHEADER, _header_list); + + // New header has been built successfully, thus mark it unchanged. + _header_changed = false; + + return dsn::error_s::ok(); +} + +dsn::error_s http_client::exec_method(const http_client::recv_callback &callback) +{ + // `curl_easy_perform` would run synchronously, thus it is safe to use the pointer to + // `callback`. + _recv_callback = &callback; + + const auto &err = process_header(); + if (!err) { + return err; + } Review Comment: How about use macro `RETURN_NOT_OK` to simplify the code? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
