This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 91594f57c3e6e58a1ecc54c913c6c21253bc713f Author: Gabor Gyimesi <[email protected]> AuthorDate: Fri Jun 30 07:32:46 2023 +0200 MINIFICPP-2027 Upgrade Google Cloud library to version 2.10.1 Closes #1572 Signed-off-by: Marton Szasz <[email protected]> --- cmake/GoogleCloudCpp.cmake | 29 +++- .../cluster/containers/FakeGcsServerContainer.py | 2 +- extensions/gcp/tests/CMakeLists.txt | 12 +- extensions/gcp/tests/DeleteGCSObjectTests.cpp | 8 +- extensions/gcp/tests/FetchGCSObjectTests.cpp | 4 +- .../tests/GCPCredentialsControllerServiceTests.cpp | 6 +- extensions/gcp/tests/ListGCSBucketTests.cpp | 8 +- extensions/gcp/tests/PutGCSObjectTests.cpp | 132 ++++++------------ libminifi/src/utils/Environment.cpp | 2 +- .../mock-client-without-decorators.patch | 153 +++++++++++++++++++++ .../google-cloud-cpp/mock_client_target.patch | 26 ++++ 11 files changed, 256 insertions(+), 126 deletions(-) diff --git a/cmake/GoogleCloudCpp.cmake b/cmake/GoogleCloudCpp.cmake index 3efabb55c..05e1d400c 100644 --- a/cmake/GoogleCloudCpp.cmake +++ b/cmake/GoogleCloudCpp.cmake @@ -22,17 +22,36 @@ include(Crc32c) set(PATCH_FILE_1 "${CMAKE_SOURCE_DIR}/thirdparty/google-cloud-cpp/remove-find_package.patch") set(PATCH_FILE_2 "${CMAKE_SOURCE_DIR}/thirdparty/google-cloud-cpp/nlohmann_lib_as_interface.patch") -set(PC ${Bash_EXECUTABLE} -c "set -x &&\ - (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_1}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_1}\\\") &&\ - (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_2}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_2}\\\")") +set(PATCH_FILE_3 "${CMAKE_SOURCE_DIR}/thirdparty/google-cloud-cpp/mock-client-without-decorators.patch") +set(PATCH_FILE_4 "${CMAKE_SOURCE_DIR}/thirdparty/google-cloud-cpp/mock_client_target.patch") +if (SKIP_TESTS) + set(PC ${Bash_EXECUTABLE} -c "set -x &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_1}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_1}\\\") &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_2}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_2}\\\")") +else() + include(GoogleTest) + include(FetchContent) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz + URL_HASH SHA256=b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5 + ) + set(BUILD_GMOCK ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + set(PC ${Bash_EXECUTABLE} -c "set -x &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_1}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_1}\\\") &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_2}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_2}\\\") &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_3}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_3}\\\") &&\ + (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_4}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_4}\\\")") +endif() set(GOOGLE_CLOUD_CPP_ENABLE storage CACHE INTERNAL storage-api) set(GOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK OFF CACHE INTERNAL macos-openssl-check) set(BUILD_TESTING OFF CACHE INTERNAL testing-off) set(GOOGLE_CLOUD_CPP_ENABLE_WERROR OFF CACHE INTERNAL warnings-off) FetchContent_Declare(google-cloud-cpp - URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v1.37.0.tar.gz - URL_HASH SHA256=a7269b21d5e95bebff7833ebb602bcd5bcc79e82a59449cc5d5b350ff2f50bbc + URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.10.1.tar.gz + URL_HASH SHA256=dbd8ff9cdda8b991049094c41d9125438098422658c77dc1afab524c589efeda PATCH_COMMAND "${PC}") add_compile_definitions(_SILENCE_CXX20_REL_OPS_DEPRECATION_WARNING _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING CURL_STATICLIB) FetchContent_MakeAvailable(google-cloud-cpp) diff --git a/docker/test/integration/cluster/containers/FakeGcsServerContainer.py b/docker/test/integration/cluster/containers/FakeGcsServerContainer.py index 32a29cb71..96f8ab70f 100644 --- a/docker/test/integration/cluster/containers/FakeGcsServerContainer.py +++ b/docker/test/integration/cluster/containers/FakeGcsServerContainer.py @@ -30,7 +30,7 @@ class FakeGcsServerContainer(Container): logging.info('Creating and running google cloud storage server docker container...') self.client.containers.run( - "fsouza/fake-gcs-server:1.44.1", + "fsouza/fake-gcs-server:1.45.1", detach=True, name=self.name, network=self.network.name, diff --git a/extensions/gcp/tests/CMakeLists.txt b/extensions/gcp/tests/CMakeLists.txt index f12204d46..0b0ffbc2e 100644 --- a/extensions/gcp/tests/CMakeLists.txt +++ b/extensions/gcp/tests/CMakeLists.txt @@ -16,16 +16,6 @@ # specific language governing permissions and limitations # under the License. # -include(GoogleTest) -include(FetchContent) -FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz - URL_HASH SHA256=b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5 -) -set(BUILD_GMOCK ON CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(googletest) - file(GLOB GCS_TESTS "*.cpp") FOREACH(testfile ${GCS_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) @@ -41,7 +31,7 @@ FOREACH(testfile ${GCS_TESTS}) target_link_libraries(${testfilename} minifi-gcp) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) - target_link_libraries(${testfilename} gtest_main gmock) + target_link_libraries(${testfilename} gtest_main gmock mock_google_cloud_client) gtest_add_tests(TARGET "${testfilename}") ENDFOREACH() diff --git a/extensions/gcp/tests/DeleteGCSObjectTests.cpp b/extensions/gcp/tests/DeleteGCSObjectTests.cpp index 777c488cc..d3e9adebd 100644 --- a/extensions/gcp/tests/DeleteGCSObjectTests.cpp +++ b/extensions/gcp/tests/DeleteGCSObjectTests.cpp @@ -39,7 +39,7 @@ class DeleteGCSObjectMocked : public DeleteGCSObject { static constexpr const char* Description = "DeleteGCSObjectMocked"; gcs::Client getClient() const override { - return gcs::testing::ClientFromMock(mock_client_, *retry_policy_); + return gcs::testing::UndecoratedClientFromMock(mock_client_); } std::shared_ptr<gcs::testing::MockClient> mock_client_ = std::make_shared<gcs::testing::MockClient>(); }; @@ -63,7 +63,7 @@ class DeleteGCSObjectTests : public ::testing::Test { }; TEST_F(DeleteGCSObjectTests, MissingBucket) { - EXPECT_CALL(*delete_gcs_object_->mock_client_, CreateResumableSession).Times(0); + EXPECT_CALL(*delete_gcs_object_->mock_client_, CreateResumableUpload).Times(0); EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::Bucket.getName(), "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(DeleteGCSObject::Success).size()); @@ -86,9 +86,7 @@ TEST_F(DeleteGCSObjectTests, ServerGivesPermaError) { } TEST_F(DeleteGCSObjectTests, ServerGivesTransientErrors) { - EXPECT_CALL(*delete_gcs_object_->mock_client_, DeleteObject) - .WillOnce(testing::Return(TransientError())) - .WillOnce(testing::Return(TransientError())); + EXPECT_CALL(*delete_gcs_object_->mock_client_, DeleteObject).WillOnce(testing::Return(TransientError())); EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::NumberOfRetries.getName(), "1")); EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::Bucket.getName(), "bucket-from-property")); const auto& result = test_controller_.trigger("hello world", {{minifi_gcp::GCS_BUCKET_ATTR, "bucket-from-attribute"}}); diff --git a/extensions/gcp/tests/FetchGCSObjectTests.cpp b/extensions/gcp/tests/FetchGCSObjectTests.cpp index bd1bd17c8..a8c16ffcb 100644 --- a/extensions/gcp/tests/FetchGCSObjectTests.cpp +++ b/extensions/gcp/tests/FetchGCSObjectTests.cpp @@ -36,7 +36,7 @@ class FetchGCSObjectMocked : public FetchGCSObject { static constexpr const char* Description = "FetchGCSObjectMocked"; gcs::Client getClient() const override { - return gcs::testing::ClientFromMock(mock_client_, *retry_policy_); + return gcs::testing::UndecoratedClientFromMock(mock_client_); } std::shared_ptr<gcs::testing::MockClient> mock_client_ = std::make_shared<gcs::testing::MockClient>(); }; @@ -60,7 +60,7 @@ class FetchGCSObjectTests : public ::testing::Test { }; TEST_F(FetchGCSObjectTests, MissingBucket) { - EXPECT_CALL(*fetch_gcs_object_->mock_client_, CreateResumableSession).Times(0); + EXPECT_CALL(*fetch_gcs_object_->mock_client_, CreateResumableUpload).Times(0); EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::Bucket.getName(), "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(FetchGCSObject::Success).size()); diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index b26fd7155..f3f11b9c3 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -25,8 +25,8 @@ #include "rapidjson/document.h" #include "rapidjson/stream.h" #include "rapidjson/writer.h" -#include "google/cloud/internal/setenv.h" #include "DummyProcessor.h" +#include "utils/Environment.h" using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; @@ -79,7 +79,7 @@ class GCPCredentialsTests : public ::testing::Test { }; TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithoutEnv) { - google::cloud::internal::UnsetEnv("GOOGLE_APPLICATION_CREDENTIALS"); + minifi::utils::Environment::unsetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS"); plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc.getName(), toString(GCPCredentialsControllerService::CredentialsLocation::USE_DEFAULT_CREDENTIALS)); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); @@ -89,7 +89,7 @@ TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) { auto temp_directory = test_controller_.createTempDirectory(); auto path = create_mock_json_file(temp_directory); ASSERT_TRUE(path.has_value()); - google::cloud::internal::SetEnv("GOOGLE_APPLICATION_CREDENTIALS", path->string()); + minifi::utils::Environment::setEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", path->string().c_str()); plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc.getName(), toString(GCPCredentialsControllerService::CredentialsLocation::USE_DEFAULT_CREDENTIALS)); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); diff --git a/extensions/gcp/tests/ListGCSBucketTests.cpp b/extensions/gcp/tests/ListGCSBucketTests.cpp index 534df31b6..4c37c30d4 100644 --- a/extensions/gcp/tests/ListGCSBucketTests.cpp +++ b/extensions/gcp/tests/ListGCSBucketTests.cpp @@ -38,7 +38,7 @@ class ListGCSBucketMocked : public ListGCSBucket { static constexpr const char* Description = "ListGCSBucketMocked"; gcs::Client getClient() const override { - return gcs::testing::ClientFromMock(mock_client_, *retry_policy_); + return gcs::testing::UndecoratedClientFromMock(mock_client_); } std::shared_ptr<gcs::testing::MockClient> mock_client_ = std::make_shared<gcs::testing::MockClient>(); }; @@ -78,7 +78,7 @@ class ListGCSBucketTests : public ::testing::Test { }; TEST_F(ListGCSBucketTests, MissingBucket) { - EXPECT_CALL(*list_gcs_bucket_->mock_client_, CreateResumableSession).Times(0); + EXPECT_CALL(*list_gcs_bucket_->mock_client_, CreateResumableUpload).Times(0); EXPECT_THROW(test_controller_.trigger(), utils::internal::RequiredPropertyMissingException); } @@ -97,9 +97,7 @@ TEST_F(ListGCSBucketTests, ServerGivesTransientErrors) { auto return_temp_error = [](ListObjectsRequest const&) { return google::cloud::StatusOr<ListObjectsResponse>(TransientError()); }; - EXPECT_CALL(*list_gcs_bucket_->mock_client_, ListObjects) - .WillOnce(return_temp_error) - .WillOnce(return_temp_error); + EXPECT_CALL(*list_gcs_bucket_->mock_client_, ListObjects).WillOnce(return_temp_error); EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::NumberOfRetries.getName(), "1")); EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::Bucket.getName(), "bucket-from-property")); const auto& result = test_controller_.trigger(); diff --git a/extensions/gcp/tests/PutGCSObjectTests.cpp b/extensions/gcp/tests/PutGCSObjectTests.cpp index 2d0f06008..1560bf52b 100644 --- a/extensions/gcp/tests/PutGCSObjectTests.cpp +++ b/extensions/gcp/tests/PutGCSObjectTests.cpp @@ -31,8 +31,7 @@ namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; using PutGCSObject = org::apache::nifi::minifi::extensions::gcp::PutGCSObject; using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; using ResumableUploadRequest = gcs::internal::ResumableUploadRequest; -using ResumableUploadResponse = gcs::internal::ResumableUploadResponse; -using ResumableUploadSession = gcs::internal::ResumableUploadSession; +using QueryResumableUploadResponse = gcs::internal::QueryResumableUploadResponse; using ::google::cloud::storage::testing::canonical_errors::TransientError; using ::google::cloud::storage::testing::canonical_errors::PermanentError; @@ -43,7 +42,7 @@ class PutGCSObjectMocked : public PutGCSObject { static constexpr const char* Description = "PutGCSObjectMocked"; gcs::Client getClient() const override { - return gcs::testing::ClientFromMock(mock_client_, *retry_policy_); + return gcs::testing::UndecoratedClientFromMock(mock_client_); } std::shared_ptr<gcs::testing::MockClient> mock_client_ = std::make_shared<gcs::testing::MockClient>(); }; @@ -65,10 +64,6 @@ class PutGCSObjectTests : public ::testing::Test { org::apache::nifi::minifi::test::SingleProcessorTestController test_controller_{put_gcs_object_}; std::shared_ptr<minifi::core::controller::ControllerServiceNode> gcp_credentials_node_; - static auto return_upload_in_progress() { - return testing::Return(google::cloud::make_status_or(ResumableUploadResponse{"fake-url", ResumableUploadResponse::kInProgress, 0, {}, {}})); - } - static auto return_upload_done(const ResumableUploadRequest& request) { using ObjectMetadataParser = gcs::internal::ObjectMetadataParser; nlohmann::json metadata_json; @@ -79,14 +74,12 @@ class PutGCSObjectTests : public ::testing::Test { metadata_json["customerEncryption"]["encryptionAlgorithm"] = "AES256"; metadata_json["customerEncryption"]["keySha256"] = "zkeXIcAB56dkHp0z1023TQZ+mzm+fZ5JRVgmAQ3bEVE="; } - return testing::Return(google::cloud::make_status_or(ResumableUploadResponse{"fake-url", - ResumableUploadResponse::kDone, 0, - *ObjectMetadataParser::FromJson(metadata_json), {}})); + return testing::Return(google::cloud::make_status_or(QueryResumableUploadResponse{absl::nullopt, *ObjectMetadataParser::FromJson(metadata_json)})); } }; TEST_F(PutGCSObjectTests, MissingBucket) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession).Times(0); + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload).Times(0); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(PutGCSObject::Success).size()); @@ -97,16 +90,11 @@ TEST_F(PutGCSObjectTests, MissingBucket) { } TEST_F(PutGCSObjectTests, BucketFromAttribute) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_EQ("bucket-from-attribute", request.bucket_name()); - - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "${gcs.bucket}")); const auto& result = test_controller_.trigger("hello world", {{minifi_gcp::GCS_BUCKET_ATTR, "bucket-from-attribute"}}); @@ -116,15 +104,7 @@ TEST_F(PutGCSObjectTests, BucketFromAttribute) { } TEST_F(PutGCSObjectTests, ServerGivesTransientErrors) { - auto return_temp_error = [](ResumableUploadRequest const&) { - return google::cloud::StatusOr<std::unique_ptr<ResumableUploadSession>>( - TransientError()); - }; - - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce(return_temp_error) - .WillOnce(return_temp_error) - .WillOnce(return_temp_error); + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload).WillOnce(testing::Return(TransientError())); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::NumberOfRetries.getName(), "2")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); @@ -137,13 +117,7 @@ TEST_F(PutGCSObjectTests, ServerGivesTransientErrors) { } TEST_F(PutGCSObjectTests, ServerGivesPermaError) { - auto return_permanent_error = [](ResumableUploadRequest const&) { - return google::cloud::StatusOr<std::unique_ptr<ResumableUploadSession>>( - PermanentError()); - }; - - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce(return_permanent_error); + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload).WillOnce(testing::Return(PermanentError())); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); @@ -155,18 +129,14 @@ TEST_F(PutGCSObjectTests, ServerGivesPermaError) { } TEST_F(PutGCSObjectTests, NonRequiredPropertiesAreMissing) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_FALSE(request.HasOption<gcs::MD5HashValue>()); EXPECT_FALSE(request.HasOption<gcs::Crc32cChecksumValue>()); EXPECT_FALSE(request.HasOption<gcs::PredefinedAcl>()); EXPECT_FALSE(request.HasOption<gcs::IfGenerationMatch>()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); @@ -176,18 +146,14 @@ TEST_F(PutGCSObjectTests, NonRequiredPropertiesAreMissing) { } TEST_F(PutGCSObjectTests, Crc32cMD5LocationTest) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption<gcs::Crc32cChecksumValue>()); EXPECT_EQ("yZRlqg==", request.GetOption<gcs::Crc32cChecksumValue>().value()); EXPECT_TRUE(request.HasOption<gcs::MD5HashValue>()); EXPECT_EQ("XrY7u+Ae7tCTyyK7j1rNww==", request.GetOption<gcs::MD5HashValue>().value()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::MD5Hash.getName(), "${md5}")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Crc32cChecksum.getName(), "${crc32c}")); @@ -199,15 +165,11 @@ TEST_F(PutGCSObjectTests, Crc32cMD5LocationTest) { } TEST_F(PutGCSObjectTests, DontOverwriteTest) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption<gcs::IfGenerationMatch>()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::OverwriteObject.getName(), "false")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); @@ -219,15 +181,11 @@ TEST_F(PutGCSObjectTests, DontOverwriteTest) { } TEST_F(PutGCSObjectTests, ValidServerSideEncryptionTest) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption<gcs::EncryptionKey>()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::EncryptionKey.getName(), "ZW5jcnlwdGlvbl9rZXk=")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); @@ -241,7 +199,7 @@ TEST_F(PutGCSObjectTests, ValidServerSideEncryptionTest) { } TEST_F(PutGCSObjectTests, InvalidServerSideEncryptionTest) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession).Times(0); + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload).Times(0); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::EncryptionKey.getName(), "not_base64_key")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); @@ -249,15 +207,11 @@ TEST_F(PutGCSObjectTests, InvalidServerSideEncryptionTest) { } TEST_F(PutGCSObjectTests, NoContentType) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_FALSE(request.HasOption<gcs::ContentType>()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); @@ -268,16 +222,12 @@ TEST_F(PutGCSObjectTests, NoContentType) { } TEST_F(PutGCSObjectTests, ContentTypeFromAttribute) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption<gcs::ContentType>()); EXPECT_EQ("text/attribute", request.GetOption<gcs::ContentType>().value()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); @@ -288,16 +238,12 @@ TEST_F(PutGCSObjectTests, ContentTypeFromAttribute) { } TEST_F(PutGCSObjectTests, ObjectACLTest) { - EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableSession) - .WillOnce([](const ResumableUploadRequest& request) { + EXPECT_CALL(*put_gcs_object_->mock_client_, CreateResumableUpload) + .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption<gcs::PredefinedAcl>()); EXPECT_EQ(gcs::PredefinedAcl::AuthenticatedRead().value(), request.GetOption<gcs::PredefinedAcl>().value()); - auto mock_upload_session = std::make_unique<gcs::testing::MockResumableUploadSession>(); - EXPECT_CALL(*mock_upload_session, done()).WillRepeatedly(testing::Return(false)); - EXPECT_CALL(*mock_upload_session, next_expected_byte()).WillRepeatedly(testing::Return(0)); - EXPECT_CALL(*mock_upload_session, UploadChunk).WillRepeatedly(return_upload_in_progress()); - EXPECT_CALL(*mock_upload_session, UploadFinalChunk).WillOnce(return_upload_done(request)); - return google::cloud::make_status_or(std::unique_ptr<gcs::internal::ResumableUploadSession>(std::move(mock_upload_session))); + EXPECT_CALL(*put_gcs_object_->mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket.getName(), "bucket-from-property")); EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key.getName(), "object-name-from-property")); diff --git a/libminifi/src/utils/Environment.cpp b/libminifi/src/utils/Environment.cpp index e0a4d9ed1..b1724a657 100644 --- a/libminifi/src/utils/Environment.cpp +++ b/libminifi/src/utils/Environment.cpp @@ -81,7 +81,7 @@ bool Environment::setEnvironmentVariable(const char* name, const char* value, bo if (!overwrite && Environment::getEnvironmentVariable(name)) { success = true; } else { - success = SetEnvironmentVariableA(name, value); + success = SetEnvironmentVariableA(name, value) && _putenv_s(name, value) == 0; } #else int ret = setenv(name, value, static_cast<int>(overwrite)); diff --git a/thirdparty/google-cloud-cpp/mock-client-without-decorators.patch b/thirdparty/google-cloud-cpp/mock-client-without-decorators.patch new file mode 100644 index 000000000..5b5822a64 --- /dev/null +++ b/thirdparty/google-cloud-cpp/mock-client-without-decorators.patch @@ -0,0 +1,153 @@ +diff --git a/google/cloud/storage/examples/storage_client_mock_samples.cc b/google/cloud/storage/examples/storage_client_mock_samples.cc +index c34002cad1..cee48946a6 100644 +--- a/google/cloud/storage/examples/storage_client_mock_samples.cc ++++ b/google/cloud/storage/examples/storage_client_mock_samples.cc +@@ -29,7 +29,7 @@ TEST(StorageMockingSamples, MockReadObject) { + + std::shared_ptr<gcs::testing::MockClient> mock = + std::make_shared<gcs::testing::MockClient>(); +- auto client = gcs::testing::ClientFromMock(mock); ++ auto client = gcs::testing::UndecoratedClientFromMock(mock); + + std::string const text = "this is a mock http response"; + std::size_t offset = 0; +@@ -72,7 +72,7 @@ TEST(StorageMockingSamples, MockWriteObject) { + + std::shared_ptr<gcs::testing::MockClient> mock = + std::make_shared<gcs::testing::MockClient>(); +- auto client = gcs::testing::ClientFromMock(mock); ++ auto client = gcs::testing::UndecoratedClientFromMock(mock); + + gcs::ObjectMetadata expected_metadata; + +@@ -97,7 +97,7 @@ TEST(StorageMockingSamples, MockReadObjectFailure) { + + std::shared_ptr<gcs::testing::MockClient> mock = + std::make_shared<gcs::testing::MockClient>(); +- auto client = gcs::testing::ClientFromMock(mock); ++ auto client = gcs::testing::UndecoratedClientFromMock(mock); + + std::string text = "this is a mock http response"; + EXPECT_CALL(*mock, ReadObject) +@@ -130,7 +130,7 @@ TEST(StorageMockingSamples, MockWriteObjectFailure) { + + std::shared_ptr<gcs::testing::MockClient> mock = + std::make_shared<gcs::testing::MockClient>(); +- auto client = gcs::testing::ClientFromMock(mock); ++ auto client = gcs::testing::UndecoratedClientFromMock(mock); + + using gcs::internal::CreateResumableUploadResponse; + using gcs::internal::QueryResumableUploadResponse; +diff --git a/google/cloud/storage/google_cloud_cpp_storage.cmake b/google/cloud/storage/google_cloud_cpp_storage.cmake +index d475554d45..5e53e972a9 100644 +--- a/google/cloud/storage/google_cloud_cpp_storage.cmake ++++ b/google/cloud/storage/google_cloud_cpp_storage.cmake +@@ -406,6 +406,7 @@ if (BUILD_TESTING) + testing/client_unit_test.cc + testing/client_unit_test.h + testing/constants.h ++ testing/mock_client.cc + testing/mock_client.h + testing/mock_http_request.cc + testing/mock_http_request.h +diff --git a/google/cloud/storage/storage_client_testing.bzl b/google/cloud/storage/storage_client_testing.bzl +index e060351a27..5727e9f9a9 100644 +--- a/google/cloud/storage/storage_client_testing.bzl ++++ b/google/cloud/storage/storage_client_testing.bzl +@@ -35,6 +35,7 @@ storage_client_testing_hdrs = [ + + storage_client_testing_srcs = [ + "testing/client_unit_test.cc", ++ "testing/mock_client.cc", + "testing/mock_http_request.cc", + "testing/object_integration_test.cc", + "testing/random_names.cc", +diff --git a/google/cloud/storage/testing/mock_client.cc b/google/cloud/storage/testing/mock_client.cc +new file mode 100644 +index 0000000000..88cd26aba3 +--- /dev/null ++++ b/google/cloud/storage/testing/mock_client.cc +@@ -0,0 +1,29 @@ ++// Copyright 2023 Google LLC ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// https://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "google/cloud/storage/testing/mock_client.h" ++ ++namespace google { ++namespace cloud { ++namespace storage { ++namespace testing { ++ ++Client UndecoratedClientFromMock(std::shared_ptr<MockClient> mock) { ++ return internal::ClientImplDetails::CreateWithoutDecorations(std::move(mock)); ++} ++ ++} // namespace testing ++} // namespace storage ++} // namespace cloud ++} // namespace google +diff --git a/google/cloud/storage/testing/mock_client.h b/google/cloud/storage/testing/mock_client.h +index 9d71ac94b9..6a763d82c6 100644 +--- a/google/cloud/storage/testing/mock_client.h ++++ b/google/cloud/storage/testing/mock_client.h +@@ -185,7 +185,12 @@ class MockStreambuf : public internal::ObjectWriteStreambuf { + MOCK_METHOD(std::uint64_t, next_expected_byte, (), (const, override)); + }; + +-/// Create a client configured to use the given mock. ++/** ++ * Create a client configured to use the given mock. ++ * ++ * Unless you specifically need to mock the behavior of retries, prefer ++ * `UndecoratedClientFromMock()`. ++ */ + template <typename... Policies> + Client ClientFromMock(std::shared_ptr<MockClient> const& mock, + Policies&&... p) { +@@ -193,6 +198,13 @@ Client ClientFromMock(std::shared_ptr<MockClient> const& mock, + mock, std::forward<Policies>(p)...); + } + ++/** ++ * Create a client configured to use the given mock. ++ * ++ * This client does not retry on transient errors. ++ */ ++Client UndecoratedClientFromMock(std::shared_ptr<MockClient> mock); ++ + } // namespace testing + } // namespace storage + } // namespace cloud +diff --git a/google/cloud/storage/testing/remove_stale_buckets_test.cc b/google/cloud/storage/testing/remove_stale_buckets_test.cc +index 23c1962fcc..afc5e85c03 100644 +--- a/google/cloud/storage/testing/remove_stale_buckets_test.cc ++++ b/google/cloud/storage/testing/remove_stale_buckets_test.cc +@@ -72,7 +72,7 @@ TEST(CleanupStaleBucketsTest, RemoveBucketContents) { + EXPECT_CALL(*mock, GetBucketMetadata) + .WillOnce( + Return(make_status_or(BucketMetadata{}.set_name("fake-bucket")))); +- auto client = internal::ClientImplDetails::CreateWithoutDecorations(mock); ++ auto client = UndecoratedClientFromMock(mock); + auto const actual = RemoveBucketAndContents(client, "fake-bucket"); + EXPECT_STATUS_OK(actual); + } +@@ -114,7 +114,7 @@ TEST(CleanupStaleBucketsTest, RemoveStaleBuckets) { + return response; + }); + +- auto client = internal::ClientImplDetails::CreateWithoutDecorations(mock); ++ auto client = UndecoratedClientFromMock(mock); + auto const actual = RemoveStaleBuckets(client, "matching", create_time_limit); + EXPECT_STATUS_OK(actual); + } diff --git a/thirdparty/google-cloud-cpp/mock_client_target.patch b/thirdparty/google-cloud-cpp/mock_client_target.patch new file mode 100644 index 000000000..70b3af164 --- /dev/null +++ b/thirdparty/google-cloud-cpp/mock_client_target.patch @@ -0,0 +1,26 @@ +diff --git a/google/cloud/storage/google_cloud_cpp_storage.cmake b/google/cloud/storage/google_cloud_cpp_storage.cmake +index 5e53e972a9..7c1f1b6542 100644 +--- a/google/cloud/storage/google_cloud_cpp_storage.cmake ++++ b/google/cloud/storage/google_cloud_cpp_storage.cmake +@@ -399,6 +399,21 @@ install( + DESTINATION include/google/cloud/storage/testing + COMPONENT google_cloud_cpp_development) + ++add_library( ++ mock_google_cloud_client # cmake-format: sort ++ testing/mock_client.cc ++ testing/mock_client.h) ++target_link_libraries( ++ mock_google_cloud_client ++ PUBLIC google-cloud-cpp::storage ++ GTest::gmock) ++google_cloud_cpp_add_common_options(mock_google_cloud_client) ++target_include_directories( ++ mock_google_cloud_client PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}> ++ $<INSTALL_INTERFACE:include>) ++target_compile_options(mock_google_cloud_client ++ PUBLIC ${GOOGLE_CLOUD_CPP_EXCEPTIONS_FLAG}) ++ + if (BUILD_TESTING) + add_library( + storage_client_testing # cmake-format: sort
