This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 94a8480aca6d24a0c9445ac98664b3c01d41a80c Author: Gabor Gyimesi <[email protected]> AuthorDate: Thu Feb 12 17:12:47 2026 +0100 MINIFICPP-2703 Update GCP extension to use unified credentials Closes #2086 Signed-off-by: Marton Szasz <[email protected]> --- cmake/GoogleCloudCpp.cmake | 4 +-- .../GCPCredentialsControllerService.cpp | 37 +++++++--------------- .../GCPCredentialsControllerService.h | 9 +++--- extensions/gcp/processors/GCSProcessor.cpp | 14 +++++--- extensions/gcp/processors/GCSProcessor.h | 6 ++-- .../tests/GCPCredentialsControllerServiceTests.cpp | 17 ++++++---- 6 files changed, 39 insertions(+), 48 deletions(-) diff --git a/cmake/GoogleCloudCpp.cmake b/cmake/GoogleCloudCpp.cmake index 071f83d5f..0d34e7856 100644 --- a/cmake/GoogleCloudCpp.cmake +++ b/cmake/GoogleCloudCpp.cmake @@ -49,8 +49,8 @@ set(GOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK OFF CACHE INTERNAL macos-openssl set(BUILD_TESTING OFF CACHE INTERNAL testing-off) set(GOOGLE_CLOUD_CPP_ENABLE_WERROR OFF CACHE INTERNAL warnings-off) FetchContent_Declare(google-cloud-cpp - URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.38.0.tar.gz - URL_HASH SHA256=f1493b2dce9b379714342f2be7ccb483d70d13aac09d4a90ae3b4756693b72fc + URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.45.0.tar.gz + URL_HASH SHA256=3d1b5eb696832f9071bf7ef0b3f0c9fd27c1a39d5edcb8a9976c65193319fd01 PATCH_COMMAND "${PC}" SYSTEM) if (WIN32) diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp index eb9842fc4..9e93c2942 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp @@ -21,8 +21,7 @@ #include "core/Resource.h" #include "google/cloud/storage/client.h" #include "utils/ProcessorConfigUtils.h" - -namespace gcs = ::google::cloud::storage; +#include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi::extensions::gcp { @@ -30,43 +29,29 @@ void GCPCredentialsControllerService::initialize() { setSupportedProperties(Properties); } -std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createDefaultCredentials() const { - auto default_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromDefaultPaths(); - if (!default_credentials.ok()) { - logger_->log_error("{}", default_credentials.status().message()); - return nullptr; - } - return *default_credentials; -} - -std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonPath() const { +std::shared_ptr<google::cloud::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonPath() const { const auto json_path = getProperty(JsonFilePath.name); if (!json_path) { logger_->log_error("Missing or invalid {}", JsonFilePath.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(*json_path); - if (!json_path_credentials.ok()) { - logger_->log_error("{}", json_path_credentials.status().message()); + if (!utils::file::exists(*json_path)) { + logger_->log_error("JSON file for GCP credentials '{}' does not exist", *json_path); return nullptr; } - return *json_path_credentials; + + return google::cloud::MakeServiceAccountCredentials(utils::file::get_content(*json_path)); } -std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonContents() const { +std::shared_ptr<google::cloud::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonContents() const { auto json_contents = getProperty(JsonContents.name); if (!json_contents) { logger_->log_error("Missing or invalid {}", JsonContents.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonContents(*json_contents); - if (!json_path_credentials.ok()) { - logger_->log_error("{}", json_path_credentials.status().message()); - return nullptr; - } - return *json_path_credentials; + return google::cloud::MakeServiceAccountCredentials(*json_contents); } void GCPCredentialsControllerService::onEnable() { @@ -79,15 +64,15 @@ void GCPCredentialsControllerService::onEnable() { credentials_location = CredentialsLocation::USE_DEFAULT_CREDENTIALS; } if (*credentials_location == CredentialsLocation::USE_DEFAULT_CREDENTIALS) { - credentials_ = createDefaultCredentials(); + credentials_ = google::cloud::MakeGoogleDefaultCredentials(); } else if (*credentials_location == CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS) { - credentials_ = gcs::oauth2::CreateComputeEngineCredentials(); + credentials_ = google::cloud::MakeComputeEngineCredentials(); } else if (*credentials_location == CredentialsLocation::USE_JSON_FILE) { credentials_ = createCredentialsFromJsonPath(); } else if (*credentials_location == CredentialsLocation::USE_JSON_CONTENTS) { credentials_ = createCredentialsFromJsonContents(); } else if (*credentials_location == CredentialsLocation::USE_ANONYMOUS_CREDENTIALS) { - credentials_ = gcs::oauth2::CreateAnonymousCredentials(); + credentials_ = google::cloud::MakeInsecureCredentials(); } if (!credentials_) logger_->log_error("Couldn't create valid credentials"); diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h index 20d7c65b2..4f5fc219b 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h @@ -28,7 +28,7 @@ #include "core/PropertyDefinitionBuilder.h" #include "utils/Enum.h" -#include "google/cloud/storage/oauth2/credentials.h" +#include "google/cloud/credentials.h" namespace org::apache::nifi::minifi::extensions::gcp { enum class CredentialsLocation { @@ -113,12 +113,11 @@ class GCPCredentialsControllerService : public core::controller::ControllerServi [[nodiscard]] const auto& getCredentials() const { return credentials_; } protected: - [[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createDefaultCredentials() const; - [[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createCredentialsFromJsonPath() const; - [[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createCredentialsFromJsonContents() const; + [[nodiscard]] std::shared_ptr<google::cloud::Credentials> createCredentialsFromJsonPath() const; + [[nodiscard]] std::shared_ptr<google::cloud::Credentials> createCredentialsFromJsonContents() const; - std::shared_ptr<google::cloud::storage::oauth2::Credentials> credentials_; + std::shared_ptr<google::cloud::Credentials> credentials_; std::shared_ptr<core::logging::Logger> logger_ = core::logging::LoggerFactory<GCPCredentialsControllerService>::getLogger(uuid_); }; } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/GCSProcessor.cpp b/extensions/gcp/processors/GCSProcessor.cpp index ded36f8a2..91d48d64a 100644 --- a/extensions/gcp/processors/GCSProcessor.cpp +++ b/extensions/gcp/processors/GCSProcessor.cpp @@ -27,7 +27,7 @@ namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { -std::shared_ptr<google::cloud::storage::oauth2::Credentials> GCSProcessor::getCredentials(core::ProcessContext& context) const { +std::shared_ptr<google::cloud::Credentials> GCSProcessor::getCredentials(core::ProcessContext& context) const { auto gcp_credentials_controller_service = utils::parseOptionalControllerService<GCPCredentialsControllerService>(context, GCSProcessor::GCPCredentials, getUUID()); if (gcp_credentials_controller_service) { return gcp_credentials_controller_service->getCredentials(); @@ -51,10 +51,14 @@ void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio } gcs::Client GCSProcessor::getClient() const { - auto options = gcs::ClientOptions(gcp_credentials_); - if (endpoint_url_) - options.set_endpoint(*endpoint_url_); - return gcs::Client(options, *retry_policy_); + auto options = google::cloud::Options{} + .set<google::cloud::UnifiedCredentialsOption>(gcp_credentials_) + .set<google::cloud::storage::RetryPolicyOption>(retry_policy_); + + if (endpoint_url_) { + options.set<gcs::RestEndpointOption>(*endpoint_url_); + } + return gcs::Client(options); } } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/GCSProcessor.h b/extensions/gcp/processors/GCSProcessor.h index e3361a786..1ec2b6641 100644 --- a/extensions/gcp/processors/GCSProcessor.h +++ b/extensions/gcp/processors/GCSProcessor.h @@ -27,7 +27,7 @@ #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" -#include "google/cloud/storage/oauth2/credentials.h" +#include "google/cloud/credentials.h" #include "google/cloud/storage/client.h" #include "google/cloud/storage/retry_policy.h" @@ -64,10 +64,10 @@ class GCSProcessor : public core::ProcessorImpl { protected: virtual google::cloud::storage::Client getClient() const; - std::shared_ptr<google::cloud::storage::oauth2::Credentials> getCredentials(core::ProcessContext& context) const; + std::shared_ptr<google::cloud::Credentials> getCredentials(core::ProcessContext& context) const; std::optional<std::string> endpoint_url_; - std::shared_ptr<google::cloud::storage::oauth2::Credentials> gcp_credentials_; + std::shared_ptr<google::cloud::Credentials> gcp_credentials_; google::cloud::storage::RetryPolicyOption::Type retry_policy_ = std::make_shared<google::cloud::storage::LimitedErrorCountRetryPolicy>(6); }; diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index 969865a74..6c33a402e 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -80,13 +80,6 @@ class GCPCredentialsTests : public ::testing::Test { std::shared_ptr<GCPCredentialsControllerService> gcp_credentials_ = std::dynamic_pointer_cast<GCPCredentialsControllerService>(gcp_credentials_node_->getControllerServiceImplementation()); }; -TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithoutEnv) { - minifi::utils::Environment::unsetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS"); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_DEFAULT_CREDENTIALS)); - ASSERT_NO_THROW(test_controller_.runSession(plan_)); - EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); -} - TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) { auto temp_directory = test_controller_.createTempDirectory(); auto path = create_mock_json_file(temp_directory); @@ -113,6 +106,16 @@ TEST_F(GCPCredentialsTests, CredentialsFromJsonWithProperty) { EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } +TEST_F(GCPCredentialsTests, CredentialsFromJsonWithInvalidPath) { + auto temp_directory = test_controller_.createTempDirectory(); + auto path = create_mock_json_file(temp_directory); + ASSERT_TRUE(path.has_value()); + plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE)); + plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, "/invalid/path/to/credentials.json"); + ASSERT_NO_THROW(test_controller_.runSession(plan_)); + EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); +} + TEST_F(GCPCredentialsTests, CredentialsFromComputeEngineVM) { plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS)); ASSERT_NO_THROW(test_controller_.runSession(plan_));
