This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 3cf34098ad7097bb3f0c0c86bf4b230a20ca160c Author: Gabor Gyimesi <[email protected]> AuthorDate: Wed May 31 15:36:42 2023 +0200 MINIFICPP-2123 Upgrade to curl to v8.1.0 Closes #1578 Signed-off-by: Marton Szasz <[email protected]> --- cmake/BundledLibcURL.cmake | 29 ++++++++++++---------- extensions/http-curl/client/HTTPClient.cpp | 9 +++++-- extensions/http-curl/processors/InvokeHTTP.cpp | 1 + .../http-curl/tests/unit/HTTPClientTests.cpp | 15 ++++++++++- extensions/splunk/PutSplunkHTTP.cpp | 1 + thirdparty/curl/module-path.patch | 13 ++++++++++ 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/cmake/BundledLibcURL.cmake b/cmake/BundledLibcURL.cmake index 534deec4a..961c93781 100644 --- a/cmake/BundledLibcURL.cmake +++ b/cmake/BundledLibcURL.cmake @@ -17,10 +17,9 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) # Define patch step - if (WIN32) - set(PC "PATCH_COMMAND ./buildconf.bat") - endif() - + set(PATCH_FILE "${SOURCE_DIR}/thirdparty/curl/module-path.patch") + set(PC ${Bash_EXECUTABLE} -c "set -x && \ + (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE}\")") # Define byproducts if (WIN32) set(BYPRODUCT "lib/libcurl.lib") @@ -40,7 +39,7 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) -DHTTP_ONLY=ON -DCURL_DISABLE_CRYPTO_AUTH=ON -DCURL_CA_PATH=none - -DCMAKE_USE_LIBSSH2=OFF + -DCURL_USE_LIBSSH2=OFF -DCMAKE_DEBUG_POSTFIX= -DHAVE_GLIBC_STRERROR_R=1 -DHAVE_GLIBC_STRERROR_R__TRYRUN_OUTPUT="" @@ -51,9 +50,9 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) -DHAVE_FSETXATTR_5__TRYRUN_OUTPUT="" ) if (OPENSSL_OFF) - list(APPEND CURL_CMAKE_ARGS -DCMAKE_USE_OPENSSL=OFF) + list(APPEND CURL_CMAKE_ARGS -DCURL_USE_OPENSSL=OFF) else() - list(APPEND CURL_CMAKE_ARGS -DCMAKE_USE_OPENSSL=ON) + list(APPEND CURL_CMAKE_ARGS -DCURL_USE_OPENSSL=ON) endif() append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") @@ -61,12 +60,12 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) # Build project ExternalProject_Add( curl-external - URL "https://github.com/curl/curl/releases/download/curl-7_64_0/curl-7.64.0.tar.gz"; - URL_HASH "SHA256=cb90d2eb74d4e358c1ed1489f8e3af96b50ea4374ad71f143fa4595e998d81b5" + URL "https://github.com/curl/curl/releases/download/curl-8_1_0/curl-8.1.0.tar.gz"; + URL_HASH "SHA256=15c00ae8d1b535ac7a7629224d36e564ce9c296698b9297bf854dd38abf226aa" SOURCE_DIR "${BINARY_DIR}/thirdparty/curl-src" LIST_SEPARATOR % # This is needed for passing semicolon-separated lists CMAKE_ARGS ${CURL_CMAKE_ARGS} - ${PC} + PATCH_COMMAND ${PC} BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" EXCLUDE_FROM_ALL TRUE ) @@ -94,9 +93,13 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) add_library(CURL::libcurl STATIC IMPORTED) set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") add_dependencies(CURL::libcurl curl-external) - set_property(TARGET CURL::libcurl APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CURL_INCLUDE_DIRS}) - set_property(TARGET CURL::libcurl APPEND PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB Threads::Threads) + target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) + target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads) + if (APPLE) + target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") + target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") + endif() if (NOT OPENSSL_OFF) - set_property(TARGET CURL::libcurl APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::SSL OpenSSL::Crypto) + target_link_libraries(CURL::libcurl INTERFACE OpenSSL::SSL OpenSSL::Crypto) endif() endfunction(use_bundled_curl SOURCE_DIR BINARY_DIR) diff --git a/extensions/http-curl/client/HTTPClient.cpp b/extensions/http-curl/client/HTTPClient.cpp index 4d2452269..95caebe01 100644 --- a/extensions/http-curl/client/HTTPClient.cpp +++ b/extensions/http-curl/client/HTTPClient.cpp @@ -31,6 +31,7 @@ #include "range/v3/algorithm/all_of.hpp" #include "range/v3/action/transform.hpp" #include "utils/HTTPUtils.h" +#include "utils/Literals.h" using namespace std::literals::chrono_literals; @@ -234,12 +235,16 @@ std::string HTTPClient::escape(std::string string_to_escape) { } void HTTPClient::setPostFields(const std::string& input) { - curl_easy_setopt(http_session_.get(), CURLOPT_POSTFIELDSIZE, input.length()); + setPostSize(input.length()); curl_easy_setopt(http_session_.get(), CURLOPT_COPYPOSTFIELDS, input.c_str()); } void HTTPClient::setPostSize(size_t size) { - curl_easy_setopt(http_session_.get(), CURLOPT_POSTFIELDSIZE, size); + if (size > 2_GB) { + curl_easy_setopt(http_session_.get(), CURLOPT_POSTFIELDSIZE_LARGE, size); + } else { + curl_easy_setopt(http_session_.get(), CURLOPT_POSTFIELDSIZE, size); + } } void HTTPClient::setHTTPProxy(const utils::HTTPProxy &proxy) { diff --git a/extensions/http-curl/processors/InvokeHTTP.cpp b/extensions/http-curl/processors/InvokeHTTP.cpp index a7cbe5a2f..b6ba1fe02 100644 --- a/extensions/http-curl/processors/InvokeHTTP.cpp +++ b/extensions/http-curl/processors/InvokeHTTP.cpp @@ -369,6 +369,7 @@ void InvokeHTTP::onTriggerWithClient(const std::shared_ptr<core::ProcessContext> client.setRequestHeader("Content-Length", "0"); } else if (!use_chunked_encoding_) { client.setRequestHeader("Content-Length", std::to_string(flow_file->getSize())); + client.setPostSize(flow_file->getSize()); } } else { logger_->log_error("InvokeHTTP -- no resource claim"); diff --git a/extensions/http-curl/tests/unit/HTTPClientTests.cpp b/extensions/http-curl/tests/unit/HTTPClientTests.cpp index 9fae6efdf..4810cc7ef 100644 --- a/extensions/http-curl/tests/unit/HTTPClientTests.cpp +++ b/extensions/http-curl/tests/unit/HTTPClientTests.cpp @@ -24,6 +24,7 @@ #include "client/HTTPClient.h" #include "CivetServer.h" #include "ConnectionCountingServer.h" +#include "utils/BaseHTTPClient.h" using namespace std::literals::chrono_literals; @@ -126,10 +127,22 @@ TEST_CASE("HTTPClient should be reusable", "[basic]") { client.setKeepAliveProbe(KeepAliveProbeData{2s, 2s}); client.initialize("GET", "http://localhost:"; + keep_alive_server_.getPort() + "/method", nullptr); - std::vector<std::string> methods = {"GET", "GET", "POST", "GET", "GET", "POST", "POST", "POST"}; + std::vector<std::string> methods = {"GET", "GET", "PUT", "GET", "GET", "POST", "POST", "PUT"}; uint64_t request_number = 0; for (const auto& method: methods) { client.set_request_method(method); + if (method != "GET") { + auto callback = std::make_unique<org::apache::nifi::minifi::utils::HTTPUploadByteArrayInputCallback>(); + std::string content = R"({ "content": "a" })"; + callback->write(content); + client.setRequestHeader("Content-Length", std::to_string(content.size())); + client.setContentType("application/json"); + client.setUploadCallback(std::move(callback)); + } else { + client.setRequestHeader("Content-Length", std::to_string(0)); + client.setUploadCallback(nullptr); + } + REQUIRE(client.submit()); const auto& headers = client.getResponseHeaderMap(); REQUIRE(headers.contains("Response-number")); diff --git a/extensions/splunk/PutSplunkHTTP.cpp b/extensions/splunk/PutSplunkHTTP.cpp index f51f4f3af..9852b599e 100644 --- a/extensions/splunk/PutSplunkHTTP.cpp +++ b/extensions/splunk/PutSplunkHTTP.cpp @@ -102,6 +102,7 @@ void setFlowFileAsPayload(core::ProcessSession& session, session.read(flow_file, std::ref(*payload)); payload->pos = 0; client.setRequestHeader("Content-Length", std::to_string(flow_file->getSize())); + client.setPostSize(flow_file->getSize()); client.setUploadCallback(std::move(payload)); diff --git a/thirdparty/curl/module-path.patch b/thirdparty/curl/module-path.patch new file mode 100644 index 000000000..90b3a2dbe --- /dev/null +++ b/thirdparty/curl/module-path.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index a0fe76c24..e12b81b91 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -63,7 +63,7 @@ + + cmake_minimum_required(VERSION 3.7...3.16 FATAL_ERROR) + +-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") ++list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") + include(Utilities) + include(Macros) + include(CMakeDependentOption)
