Repository: nifi-minifi-cpp Updated Branches: refs/heads/master 95fa21ee8 -> 63dbb8241
MINIFI-184: Add Security Support This closes #41. Signed-off-by: Aldrin Piri <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/63dbb824 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/63dbb824 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/63dbb824 Branch: refs/heads/master Commit: 63dbb8241e851068bff54ab8cef8310cc4a22cb5 Parents: 95fa21e Author: Bin Qiu <[email protected]> Authored: Fri Oct 28 09:48:57 2016 -0700 Committer: Aldrin Piri <[email protected]> Committed: Thu Feb 9 13:52:25 2017 -0500 ---------------------------------------------------------------------- .travis.yml | 1 + CMakeLists.txt | 7 ++ README.md | 28 +++++++- conf/minifi.properties | 13 ++++ libminifi/CMakeLists.txt | 8 +++ libminifi/include/Configure.h | 6 ++ libminifi/include/FlowController.h | 29 +++++++++ libminifi/include/ResourceClaim.h | 2 +- libminifi/include/Site2SitePeer.h | 7 ++ libminifi/src/Configure.cpp | 6 ++ libminifi/src/FlowController.cpp | 110 +++++++++++++++++++++++++++++++- libminifi/src/Site2SitePeer.cpp | 65 +++++++++++++++++-- main/CMakeLists.txt | 6 +- 13 files changed, 273 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 38cfca1..91ef329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,3 +35,4 @@ addons: - uuid-dev - libxml2-dev - libleveldb-dev + - openssl http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index c8857fb..9900e2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,13 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # Search for threads find_package(Threads REQUIRED) +# Set the right openssl root path +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") +set(OPENSSL_ROOT_DIR "/usr/local/opt/openssl/") +else() +set(OPENSSL_ROOT_DIR "/usr/lib/x86_64-linux-gnu") +endif() + # Provide custom modules for the project list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 2cecff3..c78e77e 100644 --- a/README.md +++ b/README.md @@ -72,12 +72,14 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a * 1.48.0 or greater * libleveldb and libleveldb-devel * libuuid and uuid-dev +* openssl ### To run #### Libraries * libuuid * libleveldb +* libssl and libcrypto from openssl The needed dependencies can be installed with the following commands for: @@ -88,7 +90,7 @@ $ yum install cmake \ gcc gcc-c++ \ leveldb-devel leveldb \ libuuid libuuid-devel \ - boost-devel + boost-devel \ libssl-dev ``` Aptitude based Linux Distributions @@ -98,7 +100,7 @@ $ apt-get install cmake \ gcc g++ \ libleveldb-dev libleveldb1v5 \ uuid-dev uuid \ - libboost-all-dev + libboost-all-dev libssl-dev ``` OS X Using Homebrew (with XCode Command Line Tools installed) @@ -107,7 +109,7 @@ OS X Using Homebrew (with XCode Command Line Tools installed) $ brew install cmake \ leveldb \ ossp-uuid \ - boost + boost \ openssl ``` @@ -238,6 +240,26 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc Port: 10001 Host Name: localhost +### Site2Site Security Configuration + + in minifi.properties + + enable tls ssl + nifi.remote.input.secure=true + + if you want to enable client certificate base authorization + nifi.security.need.ClientAuth=true + setup the client certificate and private key PEM files + nifi.security.client.certificate=./conf/client.pem + nifi.security.client.private.key=./conf/client.pem + setup the client private key passphrase file + nifi.security.client.pass.phrase=./conf/password + setup the client CA certificate file + nifi.security.client.ca.certificate=./conf/nifi-cert.pem + + if you do not want to enable client certificate base authorization + nifi.security.need.ClientAuth=false + ### Running After completing a [build](#building), the application can be run by issuing the following from : http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/conf/minifi.properties ---------------------------------------------------------------------- diff --git a/conf/minifi.properties b/conf/minifi.properties index 245c21b..62114dc 100644 --- a/conf/minifi.properties +++ b/conf/minifi.properties @@ -23,3 +23,16 @@ nifi.bored.yield.duration=10 millis nifi.provenance.repository.directory.default=./provenance_repository nifi.provenance.repository.max.storage.time=1 MIN nifi.provenance.repository.max.storage.size=1 MB + +# Security Related Properties # +# Enable tls ssl +#nifi.remote.input.secure=true +# Enable client certificate base authorization +#nifi.security.need.ClientAuth=true +# Client certificate and private key PEM files +#nifi.security.client.certificate=./conf/client.pem +#nifi.security.client.private.key=./conf/client.pem +# Client private key passphrase file +#nifi.security.client.pass.phrase=./conf/password +# Setup the client CA certificate file +#nifi.security.client.ca.certificate=./conf/nifi-cert.pem http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 8cb3cd9..8f77ed8 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -57,3 +57,11 @@ else () message( FATAL_ERROR "LevelDB was not found. Please install LevelDB" ) endif (LEVELDB_FOUND) +# Include OpenSSL +find_package (OpenSSL REQUIRED) +if (OPENSSL_FOUND) + include_directories(${OPENSSL_INCLUDE_DIR}) + target_link_libraries (minifi ${OPENSSL_LIBRARIES}) +else () + message( FATAL_ERROR "OpenSSL was not found. Please install OpenSSL" ) +endif (OPENSSL_FOUND) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/include/Configure.h ---------------------------------------------------------------------- diff --git a/libminifi/include/Configure.h b/libminifi/include/Configure.h index 7409793..00b7742 100644 --- a/libminifi/include/Configure.h +++ b/libminifi/include/Configure.h @@ -50,6 +50,12 @@ public: static const char *nifi_provenance_repository_max_storage_time; static const char *nifi_provenance_repository_max_storage_size; static const char *nifi_provenance_repository_directory_default; + static const char *nifi_remote_input_secure; + static const char *nifi_security_need_ClientAuth; + static const char *nifi_security_client_certificate; + static const char *nifi_security_client_private_key; + static const char *nifi_security_client_pass_phrase; + static const char *nifi_security_client_ca_certificate; //! Clear the load config void clear() http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/include/FlowController.h ---------------------------------------------------------------------- diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 35bcd0c..b98393e 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -53,6 +53,9 @@ #include "ListenSyslog.h" #include "ExecuteProcess.h" #include "AppendHostInfo.h" +// OpenSSL related +#include <openssl/ssl.h> +#include <openssl/err.h> //! Default NiFi Root Group Name #define DEFAULT_ROOT_GROUP_NAME "" @@ -87,6 +90,25 @@ public: } return _flowController; } + //! passphase for the private file callback + static int pemPassWordCb(char *buf, int size, int rwflag, void *userdata) + { + std::string passphrase; + + if (Configure::getConfigure()->get(Configure::nifi_security_client_pass_phrase, passphrase)) + { + std::ifstream file(passphrase.c_str(), std::ifstream::in); + if (!file.good()) + { + memset(buf, 0, size); + return 0; + } + memset(buf, 0, size); + file.getline(buf, size); + return (int) strlen(buf); + } + return 0; + } //! Destructor virtual ~FlowController(); @@ -172,6 +194,11 @@ public: { _protocol->setSerialNumber(number); } + //! getSSLContext + SSL_CTX *getSSLContext() + { + return _ctx; + } protected: @@ -204,6 +231,8 @@ protected: //! Heart Beat //! FlowControl Protocol FlowControlProtocol *_protocol; + //! SSL context + SSL_CTX *_ctx; private: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/include/ResourceClaim.h ---------------------------------------------------------------------- diff --git a/libminifi/include/ResourceClaim.h b/libminifi/include/ResourceClaim.h index d8f9979..098b259 100644 --- a/libminifi/include/ResourceClaim.h +++ b/libminifi/include/ResourceClaim.h @@ -30,7 +30,7 @@ #include "Configure.h" //! Default content directory -#define DEFAULT_CONTENT_DIRECTORY "." +#define DEFAULT_CONTENT_DIRECTORY "./content_repository" //! ResourceClaim Class class ResourceClaim { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/include/Site2SitePeer.h ---------------------------------------------------------------------- diff --git a/libminifi/include/Site2SitePeer.h b/libminifi/include/Site2SitePeer.h index ff11637..cd082f6 100644 --- a/libminifi/include/Site2SitePeer.h +++ b/libminifi/include/Site2SitePeer.h @@ -27,6 +27,7 @@ #include <netinet/in.h> #include <arpa/inet.h> #include <fcntl.h> +#include <resolv.h> #include <netdb.h> #include <string> #include <errno.h> @@ -36,6 +37,9 @@ #include "Logger.h" #include "Configure.h" #include "Property.h" +// OpenSSL related +#include <openssl/ssl.h> +#include <openssl/err.h> class CRC32 { @@ -102,6 +106,7 @@ public: _yieldExpiration = 0; _timeOut = 30000; // 30 seconds _url = "nifi://" + _host + ":" + std::to_string(_port); + _ssl = NULL; } //! Destructor virtual ~Site2SitePeer() { Close();} @@ -355,6 +360,8 @@ private: std::atomic<uint64_t> _yieldExpiration; //! Yield Expiration per destination PortID std::map<std::string, uint64_t> _yieldExpirationPortIdMap; + //! OpenSSL connection state + SSL* _ssl; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer Site2SitePeer(const Site2SitePeer &parent); http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/src/Configure.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index ab9f43e..2652e35 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -29,6 +29,12 @@ const char *Configure::nifi_server_report_interval= "nifi.server.report.interval const char *Configure::nifi_provenance_repository_max_storage_size = "nifi.provenance.repository.max.storage.size"; const char *Configure::nifi_provenance_repository_max_storage_time = "nifi.provenance.repository.max.storage.time"; const char *Configure::nifi_provenance_repository_directory_default = "nifi.provenance.repository.directory.default"; +const char *Configure::nifi_remote_input_secure = "nifi.remote.input.secure"; +const char *Configure::nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; +const char *Configure::nifi_security_client_certificate = "nifi.security.client.certificate"; +const char *Configure::nifi_security_client_private_key = "nifi.security.client.private.key"; +const char *Configure::nifi_security_client_pass_phrase = "nifi.security.client.pass.phrase"; +const char *Configure::nifi_security_client_ca_certificate = "nifi.security.client.ca.certificate"; //! Get the config value bool Configure::get(std::string key, std::string &value) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/src/FlowController.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index bc073e0..4bbc234 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -25,6 +25,9 @@ #include <time.h> #include <chrono> #include <thread> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> #include "FlowController.h" #include "ProcessContext.h" @@ -80,9 +83,108 @@ FlowController::FlowController(std::string name) exit(1); } - std::string pathString(path); - _configurationFileName = pathString; - _logger->log_info("FlowController NiFi Configuration file %s", pathString.c_str()); + std::string pathString(path); + _configurationFileName = pathString; + _logger->log_info("FlowController NiFi Configuration file %s", pathString.c_str()); + + // Create the content repo directory if needed + struct stat contentDirStat; + + if (stat(DEFAULT_CONTENT_DIRECTORY, &contentDirStat) != -1 && S_ISDIR(contentDirStat.st_mode)) + { + path = realpath(DEFAULT_CONTENT_DIRECTORY, full_path); + _logger->log_info("FlowController content directory %s", full_path); + } + else + { + if (mkdir(DEFAULT_CONTENT_DIRECTORY, 0777) == -1) + { + _logger->log_error("FlowController content directory creation failed"); + exit(1); + } + } + + std::string secureStr; + bool isSecure = false; + if (_configure->get(Configure::nifi_remote_input_secure, secureStr)) + { + Property::StringToBool(secureStr, isSecure); + } + std::string clientAuthStr; + bool needClientCert = true; + if (!(_configure->get(Configure::nifi_security_need_ClientAuth, clientAuthStr) + && Property::StringToBool(clientAuthStr, needClientCert))) + { + needClientCert = true; + } + _logger->log_info("Site2Site secure setting is %d, needClientCert %d", isSecure, needClientCert); + _ctx = NULL; + if (isSecure) + { + // create SSL context + SSL_library_init(); + const SSL_METHOD *method; + + OpenSSL_add_all_algorithms(); + SSL_load_error_strings(); + method = TLSv1_2_client_method(); + _ctx = SSL_CTX_new(method); + if ( _ctx == NULL ) + { + ERR_print_errors_fp(stderr); + _logger->log_error("Could not create SSL context, Exiting."); + exit(1); + } + if (needClientCert) + { + std::string certificate; + std::string privatekey; + std::string passphrase; + std::string caCertificate; + + if (!(_configure->get(Configure::nifi_security_client_certificate, certificate) && + _configure->get(Configure::nifi_security_client_private_key, privatekey))) + { + _logger->log_error("Certificate and Private Key PEM file not configured, Exiting."); + exit(1); + } + // load certificates and private key in PEM format + if ( SSL_CTX_use_certificate_file(_ctx, certificate.c_str(), SSL_FILETYPE_PEM) <= 0 ) + { + ERR_print_errors_fp(stderr); + _logger->log_error("Could not create load certificate, Exiting."); + exit(1); + } + if (_configure->get(Configure::nifi_security_client_pass_phrase, passphrase)) + { + // if the private key has passphase + SSL_CTX_set_default_passwd_cb(_ctx, FlowController::pemPassWordCb); + } + if ( SSL_CTX_use_PrivateKey_file(_ctx, privatekey.c_str(), SSL_FILETYPE_PEM) <= 0 ) + { + ERR_print_errors_fp(stderr); + _logger->log_error("Could not create load private key, Exiting."); + exit(1); + } + /* verify private key */ + if ( !SSL_CTX_check_private_key(_ctx) ) + { + _logger->log_error("Private key does not match the public certificate"); + exit(1); + } + /* load CA certificates */ + if (_configure->get(Configure::nifi_security_client_ca_certificate, caCertificate)) + { + if (!SSL_CTX_load_verify_locations(_ctx, caCertificate.c_str(), 0)) + { + _logger->log_error("Can not load CA certificate, Exiting."); + exit(1); + } + } + + _logger->log_info("Load/Verify Client Certificate OK."); + } + } // Create repos for flow record and provenance _provenanceRepo = new ProvenanceRepository(); @@ -97,6 +199,8 @@ FlowController::~FlowController() unload(); delete _protocol; delete _provenanceRepo; + if (_ctx) + SSL_CTX_free(_ctx); } bool FlowController::isRunning() http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/libminifi/src/Site2SitePeer.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/Site2SitePeer.cpp b/libminifi/src/Site2SitePeer.cpp index 03e662e..fb20767 100644 --- a/libminifi/src/Site2SitePeer.cpp +++ b/libminifi/src/Site2SitePeer.cpp @@ -26,6 +26,7 @@ #include <netinet/tcp.h> #include <iostream> #include "Site2SitePeer.h" +#include "FlowController.h" //! CRC tables std::atomic<bool> CRC32::tableInit(false); @@ -136,6 +137,28 @@ bool Site2SitePeer::Open() return false; } + // OpenSSL init + SSL_CTX *ctx = FlowController::getFlowController()->getSSLContext(); + if (ctx) + { + // we have s2s secure config + this->_ssl = SSL_new(ctx); + SSL_set_fd(_ssl, sock); + if (SSL_connect(_ssl) == -1) + { + _logger->log_error("SSL socket connect failed to %s %d", host, port); + SSL_free(_ssl); + _ssl = NULL; + close(sock); + this->yield(); + return false; + } + else + { + _logger->log_info("SSL socket connect success to %s %d", host, port); + } + } + _logger->log_info("Site2Site Peer socket %d connect to server %s port %d success", sock, host, port); _socket = sock; @@ -155,6 +178,11 @@ void Site2SitePeer::Close() { if (_socket) { + if (_ssl) + { + SSL_free(_ssl); + _ssl = NULL; + } _logger->log_info("Site2Site Peer socket %d close", _socket); close(_socket); _socket = 0; @@ -173,9 +201,12 @@ int Site2SitePeer::sendData(uint8_t *buf, int buflen, CRC32 *crc) while (bytes < buflen) { - ret = send(_socket, buf+bytes, buflen-bytes, 0); + if (!_ssl) + ret = send(_socket, buf+bytes, buflen-bytes, 0); + else + ret = SSL_write(_ssl, buf+bytes, buflen-bytes); //check for errors - if (ret == -1) + if (ret < 0) { _logger->log_error("Site2Site Peer socket %d send failed %s", _socket, strerror(errno)); Close(); @@ -204,6 +235,9 @@ int Site2SitePeer::Select(int msec) tv.tv_sec = msec/1000; tv.tv_usec = (msec % 1000) * 1000; + if (_ssl && SSL_pending(_ssl)) + return 1; + if (msec > 0) retval = select(fd+1, &fds, NULL, NULL, &tv); else @@ -237,12 +271,29 @@ int Site2SitePeer::readData(uint8_t *buf, int buflen, CRC32 *crc) Close(); return status; } - status = recv(_socket, buf, buflen, 0); - if (status <= 0) + if (!_ssl) { - Close(); - // this->yield(); - return status; + status = recv(_socket, buf, buflen, 0); + if (status <= 0) + { + Close(); + return status; + } + } + else + { + // for SSL, wait for the TLS IO is completed + int sslStatus; + do { + status = SSL_read(_ssl, buf, buflen); + sslStatus = SSL_get_error(_ssl, status); + } + while (status < 0 && sslStatus == SSL_ERROR_WANT_READ); + if (status <= 0) + { + Close(); + return status; + } } buflen -= status; buf += status; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/63dbb824/main/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 82b5102..e27974b 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -39,8 +39,12 @@ endif() # Include UUID find_package(UUID REQUIRED) +# Include OpenSSL +find_package(OpenSSL REQUIRED) +include_directories(${OPENSSL_INCLUDE_DIR}) + # Link against minifi, yaml-cpp, uuid, and leveldb -target_link_libraries(minifiexe minifi yaml-cpp ${UUID_LIBRARIES} ${LEVELDB_LIBRARIES}) +target_link_libraries(minifiexe minifi yaml-cpp ${UUID_LIBRARIES} ${LEVELDB_LIBRARIES} ${OPENSSL_LIBRARIES}) set_target_properties(minifiexe PROPERTIES OUTPUT_NAME minifi)
