Repository: nifi-minifi-cpp Updated Branches: refs/heads/master 1a1716dc6 -> 0a2d25c48
MINIFI-324 - More flexible uid generation This closes #115. Signed-off-by: Marc Parisi <phroc...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/0a2d25c4 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/0a2d25c4 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/0a2d25c4 Branch: refs/heads/master Commit: 0a2d25c481cfbf5b900f4208f661634fc3518d28 Parents: 1a1716d Author: Bryan Rosander <brosan...@apache.org> Authored: Wed Jun 21 14:29:03 2017 -0400 Committer: Marc Parisi <phroc...@apache.org> Committed: Fri Jun 23 11:15:38 2017 -0400 ---------------------------------------------------------------------- CMakeLists.txt | 2 +- README.md | 64 ++++--- conf/minifi-uid.properties | 28 +++ libminifi/CMakeLists.txt | 4 +- libminifi/include/FlowController.h | 2 + libminifi/include/FlowFileRecord.h | 5 +- libminifi/include/ResourceClaim.h | 9 +- libminifi/include/Site2SiteClientProtocol.h | 6 +- libminifi/include/core/Core.h | 8 +- libminifi/include/core/FlowFile.h | 4 +- libminifi/include/core/ProcessGroup.h | 2 + .../include/core/logging/LoggerConfiguration.h | 6 +- libminifi/include/core/yaml/YamlConfiguration.h | 4 +- libminifi/include/processors/InvokeHTTP.h | 2 + libminifi/include/processors/PutFile.h | 2 + libminifi/include/properties/Properties.h | 3 +- libminifi/include/provenance/Provenance.h | 24 +-- libminifi/include/utils/Id.h | 77 ++++++++ libminifi/src/FlowController.cpp | 4 +- libminifi/src/FlowFileRecord.cpp | 11 +- libminifi/src/ResourceClaim.cpp | 18 +- libminifi/src/Site2SiteClientProtocol.cpp | 5 +- libminifi/src/core/Core.cpp | 4 + libminifi/src/core/FlowFile.cpp | 10 +- libminifi/src/core/ProcessGroup.cpp | 4 +- .../src/core/logging/LoggerConfiguration.cpp | 2 +- libminifi/src/core/yaml/YamlConfiguration.cpp | 4 +- libminifi/src/processors/InvokeHTTP.cpp | 4 +- libminifi/src/processors/PutFile.cpp | 4 +- libminifi/src/provenance/Provenance.cpp | 19 +- libminifi/src/utils/Id.cpp | 154 ++++++++++++++++ libminifi/test/TestBase.h | 4 +- libminifi/test/unit/IdTests.cpp | 174 +++++++++++++++++++ main/MiNiFiMain.cpp | 10 +- 34 files changed, 585 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index fdae7ea..454908e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,7 +116,7 @@ set(CPACK_SOURCE_PACKAGE_FILE_NAME "${ASSEMBLY_BASE_NAME}-source") set(CPACK_SOURCE_IGNORE_FILES "/docs/generated/;/build/;~$;${CPACK_SOURCE_IGNORE_FILES};${CMAKE_SOURCE_DIR}/.git/;${CMAKE_SOURCE_DIR}/.idea/;${CMAKE_SOURCE_DIR}/cmake-build-debug/") # Generate binary assembly -install(FILES conf/minifi.properties conf/minifi-log.properties conf/config.yml +install(FILES conf/minifi.properties conf/minifi-log.properties conf/minifi-uid.properties conf/config.yml DESTINATION conf COMPONENT bin) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 0fb25f3..b7f532c 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a * libuuid * libleveldb * libcurl -* libssl and libcrypto from openssl +* libssl and libcrypto from openssl The needed dependencies can be installed with the following commands for: @@ -260,7 +260,7 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc - name: TransferFilesToRPG id: 471deef6-2a6e-4a7d-912a-81cc17e3a207 source name: GetFile - source id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + source id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 source relationship name: success destination id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 max work queue size: 0 @@ -283,12 +283,12 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc ### Site2Site Security Configuration - in minifi.properties + in minifi.properties enable tls ssl nifi.remote.input.secure=true - if you want to enable client certificate base authorization + if you want to enable client certificate base authorization nifi.security.need.ClientAuth=true setup the client certificate and private key PEM files nifi.security.client.certificate=./conf/client.pem @@ -300,19 +300,19 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc if you do not want to enable client certificate base authorization nifi.security.need.ClientAuth=false - + ### Configuring Volatile and NO-OP Repositories - in minifi.properties - + in minifi.properties + # For Volatile Repositories: nifi.flow.repository.class.name=VolatileRepository nifi.provenance.repository.class.name=VolatileRepository - + # For NO-OP Repositories: nifi.flow.repository.class.name=NoOpRepository nifi.provenance.repository.class.name=NoOpRepository - + ### Provenance Report Add Provenance Reporting to config.yml @@ -326,14 +326,14 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc ### Http Configuration Listener - Http Configuration Listener will pull flow file configuration from the remote command control server, + Http Configuration Listener will pull flow file configuration from the remote command control server, validate and apply the new flow configuration - in minifi.properties + in minifi.properties nifi.configuration.listener.type=http nifi.configuration.listener.http.url=https://localhost:8080 - nifi.configuration.listener.pull.interval=1 sec + nifi.configuration.listener.pull.interval=1 sec nifi.configuration.listener.client.ca.certificate=./conf/nifi-cert.pem if you want to enable client certificate @@ -341,11 +341,29 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc nifi.configuration.listener.client.certificate=./conf/client.pem nifi.configuration.listener.client.private.key=./conf/client.key nifi.configuration.listener.client.pass.phrase=./conf/password - + +### UID generation + +MiNiFi needs to generate many unique identifiers in the course of operations. There are a few different uid implementations available that can be configured in minifi-uid.properties. + +Implementation for uid generation can be selected using the uid.implementation property values: +1. time - use uuid_generate_time (default option if the file or property value is missing or invalid) +2. random - use uuid_generate_random +3. uuid_default - use uuid_generate (will attempt to use uuid_generate_random and fall back to uuid_generate_time if no high quality randomness is available) +4. minifi_uid - use custom uid algorthim + +If minifi_uuid is selected MiNiFi will use a custom uid algorthim consisting of first N bits device identifier, second M bits as bottom portion of a timestamp where N + M = 64, the last 64 bits is an atomic incrementor. + +This is faster than the random uuid generator and encodes the device id and a timestamp into every value, making tracing of flowfiles, etc easier. + +It does require more configuration. uid.minifi.device.segment.bits is used to specify how many bits at the beginning to reserve for the device identifier. It also puts a limit on how many distinct devices can be used. With the default of 16 bits, there are a maximum of 65,535 unique device identifiers that can be used. The 48 bit timestamp won't repeat for almost 9,000 years. With 24 bits for the device identifier, there are a maximum of 16,777,215 unique device identifiers and the 40 bit timestamp won't repeat for about 35 years. + +Additionally, a unique hexadecimal uid.minifi.device.segment should be assigned to each MiNiFi instance. + ### Controller Services - If you need to reference a controller service in your config.yml file, use the following template. In the example, below, ControllerServiceClass is the name of the class defining the controller Service. ControllerService1 - is linked to ControllerService2, and requires the latter to be started for ControllerService1 to start. - + If you need to reference a controller service in your config.yml file, use the following template. In the example, below, ControllerServiceClass is the name of the class defining the controller Service. ControllerService1 + is linked to ControllerService2, and requires the latter to be started for ControllerService1 to start. + Controller Services: - name: ControllerService1 id: 2438e3c8-015a-1000-79ca-83af40ec1974 @@ -358,7 +376,7 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc id: 2438e3c8-015a-1000-79ca-83af40ec1992 class: ControllerServiceClass Properties: - + ### Running After completing a [build](#building), the application can be run by issuing the following from : @@ -384,25 +402,25 @@ See https://nifi.apache.org/minifi for the latest documentation. ## Contributing -We welcome all contributions to Apache MiNiFi. To make development easier, we've included +We welcome all contributions to Apache MiNiFi. To make development easier, we've included the linter for the Google Style guide. Google provides an Eclipse formatter for their style guide. It is located [here](https://github.com/google/styleguide/blob/gh-pages/eclipse-cpp-google-style.xml). -New contributions are expected to follow the Google style guide when it is reasonable. -Additionally, all new files must include a copy of the Apache License Header. +New contributions are expected to follow the Google style guide when it is reasonable. +Additionally, all new files must include a copy of the Apache License Header. MiNiFi C++ contains a dynamic loading mechanism that loads arbitrary objects. To maintain consistency of development amongst the NiFi ecosystem, it is called a class loader. If you are contributing a custom Processor or Controller Service, the mechanism to register your class into the default class loader is a pragma definition named: - + REGISTER_RESOURCE(CLASSNAME); - + To use this include REGISTER_RESOURCE(YourClassName); in your header file. The default class loader will make instnaces of YourClassName available for inclusion. Once you have completed your changes, including source code and tests, you can verify that you follow the Google style guide by running the following command: - $ make linter. + $ make linter. This will provide output for all source files. ## License http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/conf/minifi-uid.properties ---------------------------------------------------------------------- diff --git a/conf/minifi-uid.properties b/conf/minifi-uid.properties new file mode 100644 index 0000000..fb49014 --- /dev/null +++ b/conf/minifi-uid.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Implementation for uid generation. +# Valid values: +# time - use uuid_generate_time +# random - use uuid_generate_random +# uuid_default - use uuid_generate (will attempt to use uuid_generate_random and fall back to uuid_generate_time if no high quality randomness is available) +# minifi_uid - use custom uid algorthim consisting of first N bits device identifier, second M bits as bottom portion of a timestamp where N + M = 64, last 64 bits is an atomic incrementor +uid.implementation=time + +#Number of bits at beginning of uid for device segment. +uid.minifi.device.segment.bits=16 + +# Hexadecimal representation of device segment. Should be unique per instance of MiNiFi. +#uid.minifi.device.segment=09AF http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 21ec3f7..439c641 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -62,7 +62,7 @@ include_directories(../thirdparty/jsoncpp/include) include_directories(../thirdparty/concurrentqueue/) include_directories(include) -file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") +file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/utils/*.cpp" "src/*.cpp") file(GLOB SPD_SOURCES "../thirdparty/spdlog-0.13.0/include/spdlog/*") @@ -107,3 +107,5 @@ else () message( FATAL_ERROR "OpenSSL was not found. Please install OpenSSL" ) endif (OPENSSL_FOUND) endif () + +add_subdirectory(src/utils) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/FlowController.h ---------------------------------------------------------------------- diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 6ea802c..c2fef2a 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -48,6 +48,7 @@ #include "HttpConfigurationListener.h" #include "core/Property.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -325,6 +326,7 @@ class FlowController : public core::controller::ControllerServiceProvider, publi // http configuration listener object. std::unique_ptr<HttpConfigurationListener> http_configuration_listener_; std::string serial_number_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/FlowFileRecord.h ---------------------------------------------------------------------- diff --git a/libminifi/include/FlowFileRecord.h b/libminifi/include/FlowFileRecord.h index 400cedc..3d6057d 100644 --- a/libminifi/include/FlowFileRecord.h +++ b/libminifi/include/FlowFileRecord.h @@ -103,8 +103,7 @@ class FlowFileRecord : public core::FlowFile, public io::Serializable { explicit FlowFileRecord(std::shared_ptr<core::Repository> flow_repository) : FlowFile(), flow_repository_(flow_repository), - snapshot_(""), - logger_(logging::LoggerFactory<FlowFileRecord>::getLogger()) { + snapshot_("") { } // Destructor @@ -175,7 +174,7 @@ class FlowFileRecord : public core::FlowFile, public io::Serializable { // Only support pass by reference or pointer private: - std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<logging::Logger> logger_; }; } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/ResourceClaim.h ---------------------------------------------------------------------- diff --git a/libminifi/include/ResourceClaim.h b/libminifi/include/ResourceClaim.h index 426d0dc..49faed1 100644 --- a/libminifi/include/ResourceClaim.h +++ b/libminifi/include/ResourceClaim.h @@ -27,7 +27,9 @@ #include <map> #include <mutex> #include <atomic> +#include "core/Core.h" #include "properties/Configure.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -73,10 +75,6 @@ class ResourceClaim { } protected: - // A global unique identifier - uuid_t _uuid; - // A local unique identifier - uint64_t _id; // Full path to the content std::string _contentFullPath; @@ -91,8 +89,7 @@ class ResourceClaim { ResourceClaim(const ResourceClaim &parent); ResourceClaim &operator=(const ResourceClaim &parent); - // Local resource claim number - static std::atomic<uint64_t> _localResourceClaimNumber; + static utils::NonRepeatingStringGenerator non_repeating_string_generator_; }; } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/Site2SiteClientProtocol.h ---------------------------------------------------------------------- diff --git a/libminifi/include/Site2SiteClientProtocol.h b/libminifi/include/Site2SiteClientProtocol.h index 209f6b4..a987459 100644 --- a/libminifi/include/Site2SiteClientProtocol.h +++ b/libminifi/include/Site2SiteClientProtocol.h @@ -43,6 +43,7 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "io/CRCStream.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -301,7 +302,7 @@ class Transaction { char uuidStr[37]; // Generate the global UUID for the transaction - uuid_generate(_uuid); + id_generator_->generate(_uuid); uuid_unparse_lower(_uuid, uuidStr); _uuidStr = uuidStr; } @@ -363,7 +364,7 @@ class Transaction { int _transfers; // Number of content bytes uint64_t _bytes; - + static std::shared_ptr<utils::IdGenerator> id_generator_; }; /** @@ -619,6 +620,7 @@ class Site2SiteClientProtocol { // Only support pass by reference or pointer Site2SiteClientProtocol(const Site2SiteClientProtocol &parent); Site2SiteClientProtocol &operator=(const Site2SiteClientProtocol &parent); + static std::shared_ptr<utils::IdGenerator> id_generator_; }; } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/core/Core.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/Core.h b/libminifi/include/core/Core.h index f100d8b..c32eb59 100644 --- a/libminifi/include/core/Core.h +++ b/libminifi/include/core/Core.h @@ -23,6 +23,9 @@ #include <string> #include <uuid/uuid.h> #include <cxxabi.h> + +#include "utils/Id.h" + /** * namespace aliasing */ @@ -97,7 +100,7 @@ class CoreComponent { : name_(name) { if (!uuid) // Generate the global UUID for the flow record - uuid_generate(uuid_); + id_generator_->generate(uuid_); else uuid_copy(uuid_, uuid); @@ -156,6 +159,9 @@ class CoreComponent { // Connectable's name std::string name_; + + private: + static std::shared_ptr<utils::IdGenerator> id_generator_; }; namespace logging { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/core/FlowFile.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/FlowFile.h b/libminifi/include/core/FlowFile.h index a0b8dec..95b901b 100644 --- a/libminifi/include/core/FlowFile.h +++ b/libminifi/include/core/FlowFile.h @@ -267,8 +267,8 @@ class FlowFile { std::shared_ptr<core::Connectable> original_connection_; private: - // Logger - std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; } /* namespace core */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/core/ProcessGroup.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h index 4978886..410480a 100644 --- a/libminifi/include/core/ProcessGroup.h +++ b/libminifi/include/core/ProcessGroup.h @@ -34,6 +34,7 @@ #include "core/logging/Logger.h" #include "controller/ControllerServiceNode.h" #include "controller/ControllerServiceMap.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -203,6 +204,7 @@ class ProcessGroup { // Only support pass by reference or pointer ProcessGroup(const ProcessGroup &parent); ProcessGroup &operator=(const ProcessGroup &parent); + static std::shared_ptr<utils::IdGenerator> id_generator_; }; } /* namespace core */ } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/core/logging/LoggerConfiguration.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/logging/LoggerConfiguration.h b/libminifi/include/core/logging/LoggerConfiguration.h index 99d318b..aa4a1d0 100644 --- a/libminifi/include/core/logging/LoggerConfiguration.h +++ b/libminifi/include/core/logging/LoggerConfiguration.h @@ -24,11 +24,13 @@ #include <map> #include <mutex> #include <string> +#include "spdlog/spdlog.h" +#include "spdlog/formatter.h" + +#include "core/Core.h" #include "core/logging/Logger.h" #include "properties/Properties.h" -#include "spdlog/spdlog.h" -#include "spdlog/formatter.h" namespace org { namespace apache { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/core/yaml/YamlConfiguration.h ---------------------------------------------------------------------- diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h index 61bf271..e03c794 100644 --- a/libminifi/include/core/yaml/YamlConfiguration.h +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -27,6 +27,7 @@ #include "io/validation.h" #include "io/StreamFactory.h" #include "core/logging/LoggerConfiguration.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -274,8 +275,9 @@ class YamlConfiguration : public FlowConfiguration { protected: std::shared_ptr<io::StreamFactory> stream_factory_; - private: + private: std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; } /* namespace core */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/processors/InvokeHTTP.h ---------------------------------------------------------------------- diff --git a/libminifi/include/processors/InvokeHTTP.h b/libminifi/include/processors/InvokeHTTP.h index c8e0c10..d55a5be 100644 --- a/libminifi/include/processors/InvokeHTTP.h +++ b/libminifi/include/processors/InvokeHTTP.h @@ -32,6 +32,7 @@ #include "controllers/SSLContextService.h" #include "utils/ByteInputCallBack.h" #include "core/logging/LoggerConfiguration.h" +#include "utils/Id.h" #include "utils/HTTPUtils.h" namespace org { @@ -183,6 +184,7 @@ class InvokeHTTP : public core::Processor { private: std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; REGISTER_RESOURCE(InvokeHTTP) http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/processors/PutFile.h ---------------------------------------------------------------------- diff --git a/libminifi/include/processors/PutFile.h b/libminifi/include/processors/PutFile.h index d08ebbc..f67e512 100644 --- a/libminifi/include/processors/PutFile.h +++ b/libminifi/include/processors/PutFile.h @@ -26,6 +26,7 @@ #include "core/Core.h" #include "core/Resource.h" #include "core/logging/LoggerConfiguration.h" +#include "utils/Id.h" namespace org { namespace apache { @@ -99,6 +100,7 @@ class PutFile : public core::Processor { bool putFile(core::ProcessSession *session, std::shared_ptr<FlowFileRecord> flowFile, const std::string &tmpFile, const std::string &destFile); std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; REGISTER_RESOURCE(PutFile); http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/properties/Properties.h ---------------------------------------------------------------------- diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index 8797c19..1b456d0 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -27,7 +27,6 @@ #include <errno.h> #include <iostream> #include <fstream> -#include "core/Core.h" #include "core/logging/Logger.h" namespace org { @@ -90,7 +89,7 @@ class Properties { // Mutex for protection std::mutex mutex_; // Logger - std::shared_ptr<logging::Logger> logger_; + std::shared_ptr<minifi::core::logging::Logger> logger_; // Home location for this executable std::string minifi_home_; }; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/provenance/Provenance.h ---------------------------------------------------------------------- diff --git a/libminifi/include/provenance/Provenance.h b/libminifi/include/provenance/Provenance.h index 1fa4a72..1479514 100644 --- a/libminifi/include/provenance/Provenance.h +++ b/libminifi/include/provenance/Provenance.h @@ -38,6 +38,7 @@ #include "core/logging/LoggerConfiguration.h" #include "ResourceClaim.h" #include "io/Serializable.h" +#include "utils/Id.h" #include "utils/TimeUtil.h" namespace org { @@ -160,21 +161,9 @@ class ProvenanceEventRecord : protected org::apache::nifi::minifi::io::Serializa /*! * Create a new provenance event record */ - ProvenanceEventRecord(ProvenanceEventType event, std::string componentId, std::string componentType) - : logger_(logging::LoggerFactory<ProvenanceEventRecord>::getLogger()) { - _eventType = event; - _componentId = componentId; - _componentType = componentType; - _eventTime = getTimeMillis(); - char eventIdStr[37]; - // Generate the global UUID for th event - uuid_generate(_eventId); - uuid_unparse_lower(_eventId, eventIdStr); - _eventIdStr = eventIdStr; - } + ProvenanceEventRecord(ProvenanceEventType event, std::string componentId, std::string componentType); - ProvenanceEventRecord() - : logger_(logging::LoggerFactory<ProvenanceEventRecord>::getLogger()) { + ProvenanceEventRecord() { _eventTime = getTimeMillis(); } @@ -415,15 +404,12 @@ class ProvenanceEventRecord : protected org::apache::nifi::minifi::io::Serializa std::string _alternateIdentifierUri; private: - - // Logger - std::shared_ptr<logging::Logger> logger_; - // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer ProvenanceEventRecord(const ProvenanceEventRecord &parent); ProvenanceEventRecord &operator=(const ProvenanceEventRecord &parent); - + static std::shared_ptr<logging::Logger> logger_; + static std::shared_ptr<utils::IdGenerator> id_generator_; }; // Provenance Reporter http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/include/utils/Id.h ---------------------------------------------------------------------- diff --git a/libminifi/include/utils/Id.h b/libminifi/include/utils/Id.h new file mode 100644 index 0000000..a4f8239 --- /dev/null +++ b/libminifi/include/utils/Id.h @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_UTILS_ID_H_ +#define LIBMINIFI_INCLUDE_UTILS_ID_H_ + +#include <atomic> +#include <memory> +#include <string> +#include <uuid/uuid.h> + +#include "core/logging/Logger.h" +#include "properties/Properties.h" + +#define UNSIGNED_CHAR_MAX 255 +#define UUID_TIME_IMPL 0 +#define UUID_RANDOM_IMPL 1 +#define UUID_DEFAULT_IMPL 2 +#define MINIFI_UID_IMPL 3 + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +class IdGenerator { + public: + void generate(uuid_t output); + void initialize(const std::shared_ptr<Properties> & properties); + + static std::shared_ptr<IdGenerator> getIdGenerator() { + static std::shared_ptr<IdGenerator> generator = std::shared_ptr<IdGenerator>(new IdGenerator()); + return generator; + } + protected: + uint64_t getDeviceSegmentFromString(const std::string & str, int numBits); + uint64_t getRandomDeviceSegment(int numBits); + private: + IdGenerator(); + std::shared_ptr<minifi::core::logging::Logger> logger_; + int implementation_; + unsigned char deterministic_prefix_[8]; + std::atomic<uint64_t> incrementor_; +}; + +class NonRepeatingStringGenerator { + public: + NonRepeatingStringGenerator(); + std::string generate() { + return prefix_ + std::to_string(incrementor_++); + } + private: + std::atomic<uint64_t> incrementor_; + std::string prefix_; +}; + +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_UTILS_ID_H_ */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/FlowController.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index fd75fdd..2c84811 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -47,6 +47,8 @@ namespace apache { namespace nifi { namespace minifi { +std::shared_ptr<utils::IdGenerator> FlowController::id_generator_ = utils::IdGenerator::getIdGenerator(); + #define DEFAULT_CONFIG_NAME "conf/flow.yml" FlowController::FlowController(std::shared_ptr<core::Repository> provenance_repo, std::shared_ptr<core::Repository> flow_file_repo, std::shared_ptr<Configure> configure, @@ -74,7 +76,7 @@ FlowController::FlowController(std::shared_ptr<core::Repository> provenance_repo if (IsNullOrEmpty(configuration_)) { throw std::runtime_error("Must supply a configuration."); } - uuid_generate(uuid_); + id_generator_->generate(uuid_); setUUID(uuid_); // Setup the default values http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/FlowFileRecord.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/FlowFileRecord.cpp b/libminifi/src/FlowFileRecord.cpp index 0ee33b3..12711a9 100644 --- a/libminifi/src/FlowFileRecord.cpp +++ b/libminifi/src/FlowFileRecord.cpp @@ -37,13 +37,12 @@ namespace apache { namespace nifi { namespace minifi { +std::shared_ptr<logging::Logger> FlowFileRecord::logger_ = logging::LoggerFactory<FlowFileRecord>::getLogger(); std::atomic<uint64_t> FlowFileRecord::local_flow_seq_number_(0); FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository, std::map<std::string, std::string> attributes, std::shared_ptr<ResourceClaim> claim) : FlowFile(), - flow_repository_(flow_repository), - logger_(logging::LoggerFactory<FlowFileRecord>::getLogger()) { - + flow_repository_(flow_repository) { id_ = local_flow_seq_number_.load(); claim_ = claim; // Increase the local ID for the flow record @@ -68,8 +67,7 @@ FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository, std::shared_ptr<core::FlowFile> &event, const std::string &uuidConnection) : FlowFile(), snapshot_(""), - flow_repository_(flow_repository), - logger_(logging::LoggerFactory<FlowFileRecord>::getLogger()) { + flow_repository_(flow_repository) { entry_date_ = event->getEntryDate(); lineage_start_date_ = event->getlineageStartDate(); lineage_Identifiers_ = event->getlineageIdentifiers(); @@ -88,8 +86,7 @@ FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository : FlowFile(), uuid_connection_(""), snapshot_(""), - flow_repository_(flow_repository), - logger_(logging::LoggerFactory<FlowFileRecord>::getLogger()) { + flow_repository_(flow_repository) { } FlowFileRecord::~FlowFileRecord() { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/ResourceClaim.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/ResourceClaim.cpp b/libminifi/src/ResourceClaim.cpp index 818e99b..1a9f2fe 100644 --- a/libminifi/src/ResourceClaim.cpp +++ b/libminifi/src/ResourceClaim.cpp @@ -32,26 +32,16 @@ namespace apache { namespace nifi { namespace minifi { -std::atomic<uint64_t> ResourceClaim::_localResourceClaimNumber(0); +utils::NonRepeatingStringGenerator ResourceClaim::non_repeating_string_generator_; char *ResourceClaim::default_directory_path = const_cast<char*>(DEFAULT_CONTENT_DIRECTORY); ResourceClaim::ResourceClaim(const std::string contentDirectory) - : _id(_localResourceClaimNumber.load()), - _flowFileRecordOwnedCount(0), + : _flowFileRecordOwnedCount(0), logger_(logging::LoggerFactory<ResourceClaim>::getLogger()) { - - char uuidStr[37]; - - // Generate the global UUID for the resource claim - uuid_generate(_uuid); - // Increase the local ID for the resource claim - ++_localResourceClaimNumber; - uuid_unparse_lower(_uuid, uuidStr); // Create the full content path for the content - _contentFullPath = contentDirectory + "/" + uuidStr; - - logger_->log_debug("Resource Claim created %s", _contentFullPath.c_str()); + _contentFullPath = contentDirectory + "/" + non_repeating_string_generator_.generate(); + logger_->log_debug("Resource Claim created %s", _contentFullPath); } } /* namespace minifi */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/Site2SiteClientProtocol.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/Site2SiteClientProtocol.cpp b/libminifi/src/Site2SiteClientProtocol.cpp index 08ee665..5e0637d 100644 --- a/libminifi/src/Site2SiteClientProtocol.cpp +++ b/libminifi/src/Site2SiteClientProtocol.cpp @@ -36,6 +36,9 @@ namespace apache { namespace nifi { namespace minifi { +std::shared_ptr<utils::IdGenerator> Site2SiteClientProtocol::id_generator_ = utils::IdGenerator::getIdGenerator(); +std::shared_ptr<utils::IdGenerator> Transaction::id_generator_ = utils::IdGenerator::getIdGenerator(); + bool Site2SiteClientProtocol::establish() { if (_peerState != IDLE) { logger_->log_error("Site2Site peer state is not idle while try to establish"); @@ -212,7 +215,7 @@ bool Site2SiteClientProtocol::handShake() { logger_->log_info("Site2Site Protocol Perform hand shake with destination port %s", _portIdStr.c_str()); uuid_t uuid; // Generate the global UUID for the com identify - uuid_generate(uuid); + id_generator_->generate(uuid); char uuidStr[37]; uuid_unparse_lower(uuid, uuidStr); _commsIdentifier = uuidStr; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/core/Core.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/Core.cpp b/libminifi/src/core/Core.cpp index cc6445d..304d4ce 100644 --- a/libminifi/src/core/Core.cpp +++ b/libminifi/src/core/Core.cpp @@ -17,13 +17,17 @@ */ #include "core/Core.h" +#include <memory> #include <string> + namespace org { namespace apache { namespace nifi { namespace minifi { namespace core { +std::shared_ptr<utils::IdGenerator> CoreComponent::id_generator_ = utils::IdGenerator::getIdGenerator(); + // Set UUID void CoreComponent::setUUID(uuid_t uuid) { uuid_copy(uuid_, uuid); http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/core/FlowFile.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/FlowFile.cpp b/libminifi/src/core/FlowFile.cpp index af73b91..d9057c5 100644 --- a/libminifi/src/core/FlowFile.cpp +++ b/libminifi/src/core/FlowFile.cpp @@ -21,12 +21,17 @@ #include <string> #include <set> #include "core/logging/LoggerConfiguration.h" +#include "utils/Id.h" + namespace org { namespace apache { namespace nifi { namespace minifi { namespace core { +std::shared_ptr<utils::IdGenerator> FlowFile::id_generator_ = utils::IdGenerator::getIdGenerator(); +std::shared_ptr<logging::Logger> FlowFile::logger_ = logging::LoggerFactory<FlowFile>::getLogger(); + FlowFile::FlowFile() : size_(0), id_(0), @@ -38,15 +43,14 @@ FlowFile::FlowFile() claim_(nullptr), marked_delete_(false), connection_(nullptr), - original_connection_(), - logger_(logging::LoggerFactory<FlowFile>::getLogger()) { + original_connection_() { entry_date_ = getTimeMillis(); lineage_start_date_ = entry_date_; char uuidStr[37]; // Generate the global UUID for the flow record - uuid_generate(uuid_); + id_generator_->generate(uuid_); uuid_unparse_lower(uuid_, uuidStr); uuid_str_ = uuidStr; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/core/ProcessGroup.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp index 7ac139b..2cf3db0 100644 --- a/libminifi/src/core/ProcessGroup.cpp +++ b/libminifi/src/core/ProcessGroup.cpp @@ -37,6 +37,8 @@ namespace nifi { namespace minifi { namespace core { +std::shared_ptr<utils::IdGenerator> ProcessGroup::id_generator_ = utils::IdGenerator::getIdGenerator(); + ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, int version, ProcessGroup *parent) : logger_(logging::LoggerFactory<ProcessGroup>::getLogger()), @@ -46,7 +48,7 @@ ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, parent_process_group_(parent) { if (!uuid) // Generate the global UUID for the flow record - uuid_generate(uuid_); + id_generator_->generate(uuid_); else uuid_copy(uuid_, uuid); http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/core/logging/LoggerConfiguration.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/logging/LoggerConfiguration.cpp b/libminifi/src/core/logging/LoggerConfiguration.cpp index 89e6a1b..c06239b 100644 --- a/libminifi/src/core/logging/LoggerConfiguration.cpp +++ b/libminifi/src/core/logging/LoggerConfiguration.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "core/logging/LoggerConfiguration.h" #include <algorithm> #include <vector> #include <queue> @@ -26,7 +27,6 @@ #include <string> #include "core/Core.h" -#include "core/logging/LoggerConfiguration.h" #include "utils/StringUtils.h" #include "spdlog/spdlog.h" http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/core/yaml/YamlConfiguration.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp index 44aec12..c28ec70 100644 --- a/libminifi/src/core/yaml/YamlConfiguration.cpp +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -29,6 +29,8 @@ namespace nifi { namespace minifi { namespace core { +std::shared_ptr<utils::IdGenerator> YamlConfiguration::id_generator_ = utils::IdGenerator::getIdGenerator(); + core::ProcessGroup *YamlConfiguration::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { uuid_t uuid; int64_t version = 0; @@ -610,7 +612,7 @@ std::string YamlConfiguration::getOrGenerateId(YAML::Node *yamlNode, const std:: } } else { uuid_t uuid; - uuid_generate(uuid); + id_generator_->generate(uuid); char uuid_str[37]; uuid_unparse(uuid, uuid_str); id = uuid_str; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/processors/InvokeHTTP.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/processors/InvokeHTTP.cpp b/libminifi/src/processors/InvokeHTTP.cpp index c636201..d5097bb 100644 --- a/libminifi/src/processors/InvokeHTTP.cpp +++ b/libminifi/src/processors/InvokeHTTP.cpp @@ -49,6 +49,8 @@ namespace nifi { namespace minifi { namespace processors { +std::shared_ptr<utils::IdGenerator> InvokeHTTP::id_generator_ = utils::IdGenerator::getIdGenerator(); + const char *InvokeHTTP::ProcessorName = "InvokeHTTP"; core::Property InvokeHTTP::Method("HTTP Method", "HTTP request method (GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS). " @@ -235,7 +237,7 @@ inline bool InvokeHTTP::matches(const std::string &value, const std::string &sre std::string InvokeHTTP::generateId() { uuid_t txId; - uuid_generate(txId); + id_generator_->generate(txId); char uuidStr[37]; uuid_unparse_lower(txId, uuidStr); return uuidStr; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/processors/PutFile.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/processors/PutFile.cpp b/libminifi/src/processors/PutFile.cpp index 0aba3d7..d72c56a 100644 --- a/libminifi/src/processors/PutFile.cpp +++ b/libminifi/src/processors/PutFile.cpp @@ -38,6 +38,8 @@ namespace nifi { namespace minifi { namespace processors { +std::shared_ptr<utils::IdGenerator> PutFile::id_generator_ = utils::IdGenerator::getIdGenerator(); + core::Property PutFile::Directory("Output Directory", "The output directory to which to put files", "."); core::Property PutFile::ConflictResolution("Conflict Resolution Strategy", "Indicates what should happen when a file with the same name already exists in the output directory", CONFLICT_RESOLUTION_STRATEGY_FAIL); @@ -87,7 +89,7 @@ void PutFile::onTrigger(core::ProcessContext *context, core::ProcessSession *ses // Generate a safe (universally-unique) temporary filename on the same partition char tmpFileUuidStr[37]; uuid_t tmpFileUuid; - uuid_generate(tmpFileUuid); + id_generator_->generate(tmpFileUuid); uuid_unparse_lower(tmpFileUuid, tmpFileUuidStr); std::stringstream tmpFileSs; tmpFileSs << directory_ << "/." << filename << "." << tmpFileUuidStr; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/provenance/Provenance.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/provenance/Provenance.cpp b/libminifi/src/provenance/Provenance.cpp index ff6a149..8686a58 100644 --- a/libminifi/src/provenance/Provenance.cpp +++ b/libminifi/src/provenance/Provenance.cpp @@ -16,14 +16,16 @@ * limitations under the License. */ +#include "provenance/Provenance.h" + #include <arpa/inet.h> #include <cstdint> #include <memory> #include <string> #include <vector> + #include "io/DataStream.h" #include "io/Serializable.h" -#include "provenance/Provenance.h" #include "core/logging/Logger.h" #include "core/Relationship.h" #include "FlowController.h" @@ -34,9 +36,24 @@ namespace nifi { namespace minifi { namespace provenance { +std::shared_ptr<utils::IdGenerator> ProvenanceEventRecord::id_generator_ = utils::IdGenerator::getIdGenerator(); +std::shared_ptr<logging::Logger> ProvenanceEventRecord::logger_ = logging::LoggerFactory<ProvenanceEventRecord>::getLogger(); + const char *ProvenanceEventRecord::ProvenanceEventTypeStr[REPLAY + 1] = { "CREATE", "RECEIVE", "FETCH", "SEND", "DOWNLOAD", "DROP", "EXPIRE", "FORK", "JOIN", "CLONE", "CONTENT_MODIFIED", "ATTRIBUTES_MODIFIED", "ROUTE", "ADDINFO", "REPLAY" }; +ProvenanceEventRecord::ProvenanceEventRecord(ProvenanceEventRecord::ProvenanceEventType event, std::string componentId, std::string componentType) { + _eventType = event; + _componentId = componentId; + _componentType = componentType; + _eventTime = getTimeMillis(); + char eventIdStr[37]; + // Generate the global UUID for th event + id_generator_->generate(_eventId); + uuid_unparse_lower(_eventId, eventIdStr); + _eventIdStr = eventIdStr; +} + // DeSerialize bool ProvenanceEventRecord::DeSerialize(const std::shared_ptr<core::Repository> &repo, std::string key) { std::string value; http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/src/utils/Id.cpp ---------------------------------------------------------------------- diff --git a/libminifi/src/utils/Id.cpp b/libminifi/src/utils/Id.cpp new file mode 100644 index 0000000..e74652e --- /dev/null +++ b/libminifi/src/utils/Id.cpp @@ -0,0 +1,154 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/Id.h" + +#include <inttypes.h> +#include <uuid/uuid.h> + +#include <algorithm> +#include <chrono> +#include <cmath> +#include <memory> +#include <string> + +#include "core/logging/LoggerConfiguration.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +uint64_t timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); + +NonRepeatingStringGenerator::NonRepeatingStringGenerator() : prefix_((std::to_string(timestamp) + "-")) {} + +IdGenerator::IdGenerator() : implementation_(UUID_TIME_IMPL), logger_(logging::LoggerFactory<IdGenerator>::getLogger()), incrementor_(0) { +} + +uint64_t IdGenerator::getDeviceSegmentFromString(const std::string& str, int numBits) { + uint64_t deviceSegment = 0; + for (int i = 0; i < str.length(); i++) { + unsigned char c = toupper(str[i]); + if (c >= '0' && c <= '9') { + deviceSegment = deviceSegment + (c - '0'); + } else if (c >= 'A' && c <= 'F') { + deviceSegment = deviceSegment + (c - 'A' + 10); + } else { + logger_->log_error("Expected hex char (0-9, A-F). Got %c.", c); + } + deviceSegment = deviceSegment << 4; + } + deviceSegment <<= 64 - (4 * (str.length() + 1)); + deviceSegment >>= 64 - numBits; + logger_->log_debug("Using user defined device segment: %" PRIx64, deviceSegment); + deviceSegment <<= 64 - numBits; + return deviceSegment; +} + +uint64_t IdGenerator::getRandomDeviceSegment(int numBits) { + uint64_t deviceSegment = 0; + uuid_t random_uuid; + for (int word = 0; word < 2; word++) { + uuid_generate_random(random_uuid); + for (int i = 0; i < 4; i++) { + deviceSegment += random_uuid[i]; + deviceSegment <<= 8; + } + } + deviceSegment >>= 64 - numBits; + logger_->log_debug("Using random device segment: %" PRIx64, deviceSegment); + deviceSegment <<= 64 - numBits; + return deviceSegment; +} + +void IdGenerator::initialize(const std::shared_ptr<Properties> & properties) { + std::string implementation_str; + implementation_ = UUID_TIME_IMPL; + if (properties->get("uid.implementation", implementation_str)) { + std::transform(implementation_str.begin(), implementation_str.end(), implementation_str.begin(), ::tolower); + if ("random" == implementation_str) { + logger_->log_debug("Using uuid_generate_random for uids."); + implementation_ = UUID_RANDOM_IMPL; + } else if ("uuid_default" == implementation_str) { + logger_->log_debug("Using uuid_generate for uids."); + implementation_ = UUID_DEFAULT_IMPL; + } else if ("minifi_uid" == implementation_str) { + logger_->log_debug("Using minifi uid implementation for uids"); + implementation_ = MINIFI_UID_IMPL; + + uint64_t timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); + int device_bits = properties->getInt("uid.minifi.device.segment.bits", 16); + std::string device_segment; + uint64_t prefix = timestamp; + if (device_bits > 0) { + if (properties->get("uid.minifi.device.segment", device_segment)) { + prefix = getDeviceSegmentFromString(device_segment, device_bits); + } else { + logger_->log_warn("uid.minifi.device.segment not specified, generating random device segment"); + prefix = getRandomDeviceSegment(device_bits); + } + timestamp <<= device_bits; + timestamp >>= device_bits; + prefix = prefix + timestamp; + logger_->log_debug("Using minifi uid prefix: %16" PRIx64, prefix); + } + for (int i = 0; i < 8; i++) { + unsigned char prefix_element = (prefix >> ((7 - i) * 8)) & UNSIGNED_CHAR_MAX; + deterministic_prefix_[i] = prefix_element; + } + incrementor_ = 0; + } else if ("time" == implementation_str) { + logger_->log_debug("Using uuid_generate_time implementation for uids."); + } else { + logger_->log_debug("Invalid value for uid.implementation (%s). Using uuid_generate_time implementation for uids.", implementation_str); + } + } else { + logger_->log_debug("Using uuid_generate_time implementation for uids."); + } +} + +void IdGenerator::generate(uuid_t output) { + switch (implementation_) { + case UUID_RANDOM_IMPL: + uuid_generate_random(output); + break; + case UUID_DEFAULT_IMPL: + uuid_generate(output); + break; + case MINIFI_UID_IMPL: { + std::memcpy(output, deterministic_prefix_, sizeof(deterministic_prefix_)); + uint64_t incrementor_value = incrementor_++; + for (int i = 8; i < 16; i++) { + output[i] = (incrementor_value >> ((15 - i) * 8)) & UNSIGNED_CHAR_MAX; + } + } + break; + default: + uuid_generate_time(output); + break; + } +} + +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/test/TestBase.h ---------------------------------------------------------------------- diff --git a/libminifi/test/TestBase.h b/libminifi/test/TestBase.h index e675043..331df08 100644 --- a/libminifi/test/TestBase.h +++ b/libminifi/test/TestBase.h @@ -30,6 +30,7 @@ #include "properties/Configure.h" #include "properties/Properties.h" #include "core/logging/LoggerConfiguration.h" +#include "utils/Id.h" #include "spdlog/sinks/ostream_sink.h" #include "spdlog/sinks/dist_sink.h" @@ -84,7 +85,7 @@ class LogTestController { bool contains(const std::ostringstream &stream, const std::string &ending) { std::string str = stream.str(); - logger_->log_info("Looking for %s in %s.", ending, str); + logger_->log_info("Looking for %s in log output.", ending); return (ending.length() > 0 && str.find(ending) != std::string::npos); } @@ -144,6 +145,7 @@ class TestController { : log(LogTestController::getInstance()) { minifi::ResourceClaim::default_directory_path = const_cast<char*>("./"); log.reset(); + utils::IdGenerator::getIdGenerator()->initialize(std::make_shared<minifi::Properties>()); } ~TestController() { http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/libminifi/test/unit/IdTests.cpp ---------------------------------------------------------------------- diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp new file mode 100644 index 0000000..946fc0c --- /dev/null +++ b/libminifi/test/unit/IdTests.cpp @@ -0,0 +1,174 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file +#include <utility> +#include <string> +#include <memory> +#include <ctime> +#include "../TestBase.h" +#include "utils/Id.h" + +TEST_CASE("Test default is time", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(std::make_shared<minifi::Properties>()); + + REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_time implementation for uids.")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test time", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "TiMe"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_time implementation for uids.")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test random", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "RaNDoM"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_random for uids.")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test uuid_default", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "UUID_default"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate for uids.")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test invalid", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "InVaLiD"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + REQUIRE(true == LogTestController::getInstance().contains("Invalid value for uid.implementation (invalid). Using uuid_generate_time implementation for uids.")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test Hex Device Segment 16 bits correct digits", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "minifi_uid"); + id_props->set("uid.minifi.device.segment", "09aF"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + uuid_t uid; + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(0 == uid[15]); + + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(1 == uid[15]); + + REQUIRE(true == LogTestController::getInstance().contains("Using user defined device segment: 9af")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test Hex Device Segment 16 bits too many digits", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "minifi_uid"); + id_props->set("uid.minifi.device.segment", "09aFee"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + uuid_t uid; + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(0 == (uid[2] & 128)); + REQUIRE(0 == uid[15]); + + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(0 == (uid[2] & 128)); + REQUIRE(1 == uid[15]); + + REQUIRE(true == LogTestController::getInstance().contains("Using user defined device segment: 9af")); + LogTestController::getInstance().reset(); +} + +TEST_CASE("Test Hex Device Segment 18 bits", "[id]") { + TestController test_controller; + + LogTestController::getInstance().setDebug<utils::IdGenerator>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::Properties>(); + id_props->set("uid.implementation", "minifi_uid"); + id_props->set("uid.minifi.device.segment.bits", "18"); + id_props->set("uid.minifi.device.segment", "09aF8"); + + std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); + generator->initialize(id_props); + + uuid_t uid; + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(128 == (uid[2] & 192)); + REQUIRE(0 == uid[15]); + + generator->generate(uid); + REQUIRE(0x09 == uid[0]); + REQUIRE(0xaf == uid[1]); + REQUIRE(128 == (uid[2] & 192)); + REQUIRE(1 == uid[15]); + + REQUIRE(true == LogTestController::getInstance().contains("Using minifi uid prefix: 9af8")); + LogTestController::getInstance().reset(); +} http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/0a2d25c4/main/MiNiFiMain.cpp ---------------------------------------------------------------------- diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp index 297313d..ba59623 100644 --- a/main/MiNiFiMain.cpp +++ b/main/MiNiFiMain.cpp @@ -38,6 +38,7 @@ #include "core/logging/LoggerConfiguration.h" #include "properties/Properties.h" #include "properties/Configure.h" +#include "utils/Id.h" #include "FlowController.h" //! Main thread sleep interval 1 second @@ -46,10 +47,10 @@ #define STOP_WAIT_TIME_MS 30*1000 //! Default YAML location #define DEFAULT_NIFI_CONFIG_YML "./conf/config.yml" -//! Default nifi properties file path +//! Default properties file paths #define DEFAULT_NIFI_PROPERTIES_FILE "./conf/minifi.properties" - #define DEFAULT_LOG_PROPERTIES_FILE "./conf/minifi-log.properties" +#define DEFAULT_UID_PROPERTIES_FILE "./conf/minifi-uid.properties" //! Define home environment variable #define MINIFI_HOME_ENV_KEY "MINIFI_HOME" @@ -123,6 +124,11 @@ int main(int argc, char **argv) { log_properties->setHome(minifiHome); log_properties->loadConfigureFile(DEFAULT_LOG_PROPERTIES_FILE); logging::LoggerConfiguration::getConfiguration().initialize(log_properties); + + std::shared_ptr<minifi::Properties> uid_properties = std::make_shared<minifi::Properties>(); + uid_properties->setHome(minifiHome); + uid_properties->loadConfigureFile(DEFAULT_UID_PROPERTIES_FILE); + utils::IdGenerator::getIdGenerator()->initialize(uid_properties); // Make a record of minifi home in the configured log file. logger->log_info("MINIFI_HOME=%s", minifiHome);
