Repository: nifi-minifi-cpp Updated Branches: refs/heads/master 81cba0ed1 -> 05ee9d339
MINIFICPP-404: Correct invalid assumption and protect against invalid protocols This closes #454. Signed off on GH by arpadboda. Signed-off-by: Marc Parisi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/05ee9d33 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/05ee9d33 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/05ee9d33 Branch: refs/heads/master Commit: 05ee9d339a7b1962c7126f340fe61ea11b5cac04 Parents: 81cba0e Author: Marc Parisi <[email protected]> Authored: Mon Dec 3 13:14:26 2018 -0500 Committer: Marc Parisi <[email protected]> Committed: Wed Dec 5 14:47:04 2018 -0500 ---------------------------------------------------------------------- .../http-curl/sitetosite/HTTPProtocol.cpp | 1 + libminifi/include/core/yaml/YamlConfiguration.h | 1 + libminifi/src/RemoteProcessorGroupPort.cpp | 16 +-- libminifi/src/core/yaml/YamlConfiguration.cpp | 7 ++ libminifi/src/sitetosite/SiteToSiteClient.cpp | 2 +- libminifi/test/resources/TestHTTPSiteToSite.yml | 1 + libminifi/test/resources/TestSite2SiteRest.yml | 1 + .../test/resources/TestSite2SiteRestSecure.yml | 1 + libminifi/test/unit/YamlConfigurationTests.cpp | 126 +++++++++++++++++++ 9 files changed, 148 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/extensions/http-curl/sitetosite/HTTPProtocol.cpp ---------------------------------------------------------------------- diff --git a/extensions/http-curl/sitetosite/HTTPProtocol.cpp b/extensions/http-curl/sitetosite/HTTPProtocol.cpp index df3bdc8..7698f50 100644 --- a/extensions/http-curl/sitetosite/HTTPProtocol.cpp +++ b/extensions/http-curl/sitetosite/HTTPProtocol.cpp @@ -117,6 +117,7 @@ int HttpSiteToSiteClient::readResponse(const std::shared_ptr<Transaction> &trans code = CONFIRM_TRANSACTION; message = std::string(client->getResponseBody().data(), client->getResponseBody().size()); } else { + logger_->log_debug("Received response code %d", client->getResponseCode()); code = UNRECOGNIZED_RESPONSE_CODE; } return 1; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/include/core/yaml/YamlConfiguration.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h index b60d384..a8c43b8 100644 --- a/libminifi/include/core/yaml/YamlConfiguration.h +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -20,6 +20,7 @@ #include "core/ProcessorConfig.h" #include "yaml-cpp/yaml.h" +#include "Exception.h" #include "core/FlowConfiguration.h" #include "sitetosite/SiteToSite.h" #include <string> http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/src/RemoteProcessorGroupPort.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index 64fff3c..0f8daec 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -120,13 +120,6 @@ void RemoteProcessorGroupPort::initialize() { relationships.insert(relation); setSupportedRelationships(relationships); - client_type_ = sitetosite::RAW; - std::string http_enabled_str; - if (configure_->get(Configure::nifi_remote_input_http, http_enabled_str)) { - if (utils::StringUtils::StringToBool(http_enabled_str, http_enabled_)) { - client_type_ = sitetosite::HTTP; - } - } logger_->log_trace("Finished initialization"); } @@ -136,6 +129,15 @@ void RemoteProcessorGroupPort::onSchedule(const std::shared_ptr<core::ProcessCon protocol_uuid_ = value; } + std::string http_enabled_str; + if (configure_->get(Configure::nifi_remote_input_http, http_enabled_str)) { + if (utils::StringUtils::StringToBool(http_enabled_str, http_enabled_)) { + if (client_type_ == sitetosite::CLIENT_TYPE::RAW) { + logger_->log_debug("Remote Input HTTP Enabled, but raw has been suggested for %s", protocol_uuid_); + } + } + } + std::string context_name; if (!context->getProperty(SSLContext.getName(), context_name) || IsNullOrEmpty(context_name)) { context_name = RPG_SSL_CONTEXT_SERVICE_NAME; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/src/core/yaml/YamlConfiguration.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp index af88082..5458597 100644 --- a/libminifi/src/core/yaml/YamlConfiguration.cpp +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -325,6 +325,12 @@ void YamlConfiguration::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, core::P } } } + } else if (transport_protocol == "RAW") { + group->setTransportProtocol(transport_protocol); + } else { + std::stringstream stream; + stream << "Invalid transport protocol " << transport_protocol; + throw minifi::Exception(ExceptionType::SITE2SITE_EXCEPTION, stream.str().c_str()); } } @@ -672,6 +678,7 @@ void YamlConfiguration::parsePortYaml(YAML::Node *portNode, core::ProcessGroup * if (!parent->getHttpProxyHost().empty()) port->setHTTPProxy(parent->getHTTPProxy()); } + // else defaults to RAW // handle port properties YAML::Node nodeVal = portNode->as<YAML::Node>(); http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/src/sitetosite/SiteToSiteClient.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/sitetosite/SiteToSiteClient.cpp b/libminifi/src/sitetosite/SiteToSiteClient.cpp index 0eafb2b..142de56 100644 --- a/libminifi/src/sitetosite/SiteToSiteClient.cpp +++ b/libminifi/src/sitetosite/SiteToSiteClient.cpp @@ -298,7 +298,7 @@ bool SiteToSiteClient::confirm(std::string transactionID) { logger_->log_debug("Site2Site transaction %s peer indicate bad checksum", transactionID); return false; } else { - logger_->log_debug("Site2Site transaction %s peer unknown respond code %d", transactionID, code); + logger_->log_debug("Site2Site transaction %s peer unknown response code %d", transactionID, code); return false; } } else { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/test/resources/TestHTTPSiteToSite.yml ---------------------------------------------------------------------- diff --git a/libminifi/test/resources/TestHTTPSiteToSite.yml b/libminifi/test/resources/TestHTTPSiteToSite.yml index 4bb9104..2bdee30 100644 --- a/libminifi/test/resources/TestHTTPSiteToSite.yml +++ b/libminifi/test/resources/TestHTTPSiteToSite.yml @@ -70,6 +70,7 @@ Remote Processing Groups: url: http://localhost:8099/nifi timeout: 30 secs yield period: 1 sec + transport protocol: HTTP Input Ports: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 name: From Node A http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/test/resources/TestSite2SiteRest.yml ---------------------------------------------------------------------- diff --git a/libminifi/test/resources/TestSite2SiteRest.yml b/libminifi/test/resources/TestSite2SiteRest.yml index 6adbc7c..fcdbff0 100644 --- a/libminifi/test/resources/TestSite2SiteRest.yml +++ b/libminifi/test/resources/TestSite2SiteRest.yml @@ -49,6 +49,7 @@ Remote Processing Groups: url: http://localhost:8033/nifi,http://localhost:8066/nifi, http://localhost:8077/nifi timeout: 30 secs yield period: 10 sec + transport protocol: RAW Input Ports: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 name: From Node A http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/test/resources/TestSite2SiteRestSecure.yml ---------------------------------------------------------------------- diff --git a/libminifi/test/resources/TestSite2SiteRestSecure.yml b/libminifi/test/resources/TestSite2SiteRestSecure.yml index 7d27969..b1a0509 100644 --- a/libminifi/test/resources/TestSite2SiteRestSecure.yml +++ b/libminifi/test/resources/TestSite2SiteRestSecure.yml @@ -49,6 +49,7 @@ Remote Processing Groups: url: https://localhost:8073/nifi,https://localhost:8072/nifi timeout: 30 secs yield period: 10 sec + transport protocol: HTTP Input Ports: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 name: From Node A http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/05ee9d33/libminifi/test/unit/YamlConfigurationTests.cpp ---------------------------------------------------------------------- diff --git a/libminifi/test/unit/YamlConfigurationTests.cpp b/libminifi/test/unit/YamlConfigurationTests.cpp index 671b59f..71ef512 100644 --- a/libminifi/test/unit/YamlConfigurationTests.cpp +++ b/libminifi/test/unit/YamlConfigurationTests.cpp @@ -190,6 +190,132 @@ TEST_CASE("Test YAML Config Processing", "[YamlConfiguration]") { } } +TEST_CASE("Test YAML v3 Invalid Type", "[YamlConfiguration3]") { + TestController test_controller; + + std::shared_ptr<core::Repository> testProvRepo = core::createRepository("provenancerepository", true); + std::shared_ptr<core::Repository> testFlowFileRepo = core::createRepository("flowfilerepository", true); + std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::Configure>(); + std::shared_ptr<minifi::io::StreamFactory> streamFactory = minifi::io::StreamFactory::getInstance(configuration); + std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>(); + core::YamlConfiguration yamlConfig(testProvRepo, testFlowFileRepo, content_repo, streamFactory, configuration); + + static const std::string TEST_CONFIG_YAML = + R"( +MiNiFi Config Version: 3 +Flow Controller: + name: Simple TailFile To RPG + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 + variable registry properties: '' +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min + implementation: org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: b0c04f28-0158-1000-0000-000000000000 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File Location: Local + File to Tail: ./logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: + tail-base-directory: + tail-mode: Single file + tailfile-lookup-frequency: 10 minutes + tailfile-maximum-age: 24 hours + tailfile-recursive-lookup: 'false' + tailfile-rolling-strategy: Fixed name +Controller Services: [] +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: b0c0c3cc-0158-1000-0000-000000000000 + name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 + source id: b0c04f28-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: ac0e798c-0158-1000-0588-cda9b944e011 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: +- id: b0c09ff0-0158-1000-0000-000000000000 + name: '' + url: http://localhost:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: WRONG + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' + local network interface: '' + Input Ports: + - id: aca664f8-0158-1000-a139-92485891d349 + name: test2 + comment: '' + max concurrent tasks: 1 + use compression: false + - id: ac0e798c-0158-1000-0588-cda9b944e011 + name: test + comment: '' + max concurrent tasks: 1 + use compression: false + Output Ports: [] +NiFi Properties Overrides: {} + )"; + std::istringstream configYamlStream(TEST_CONFIG_YAML); + + REQUIRE_THROWS_AS(yamlConfig.getYamlRoot(configYamlStream), minifi::Exception); +} + TEST_CASE("Test YAML v3 Config Processing", "[YamlConfiguration3]") { TestController test_controller;
