This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 6dd03a62019950baf9c0ff9b2a0719f2fb9e54f8 Author: Martin Zink <[email protected]> AuthorDate: Mon Jun 7 08:30:40 2021 +0200 MINIFICPP-1567 enable linter checks in extensions (part 4) enable rocksdb-repos linter enable jni linter enable script linter enable sftp linter enable sensor linter Signed-off-by: Ferenc Gerlits <[email protected]> This closes #1112 --- extensions/jni/CMakeLists.txt | 12 +- extensions/jni/ClassRegistrar.h | 7 +- extensions/jni/ExecuteJavaControllerService.h | 10 +- extensions/jni/ExecuteJavaProcessor.cpp | 11 +- extensions/jni/ExecuteJavaProcessor.h | 31 +++-- extensions/jni/JNILoader.h | 11 +- extensions/jni/JNIUtil.h | 7 +- extensions/jni/JVMCreator.h | 7 +- extensions/jni/JavaException.h | 16 +-- extensions/jni/jvm/JVMLoader.h | 31 ++--- extensions/jni/jvm/JavaClass.h | 11 +- extensions/jni/jvm/JavaControllerService.cpp | 2 +- extensions/jni/jvm/JavaControllerService.h | 12 +- extensions/jni/jvm/JavaServicer.h | 1 + extensions/jni/jvm/JniBundle.h | 10 +- extensions/jni/jvm/JniConfigurationContext.h | 9 +- extensions/jni/jvm/JniControllerServiceLookup.h | 6 +- extensions/jni/jvm/JniFlowFile.h | 4 +- extensions/jni/jvm/JniInitializationContext.h | 10 +- extensions/jni/jvm/JniLogger.h | 9 +- extensions/jni/jvm/JniMethod.h | 13 +- extensions/jni/jvm/JniProcessContext.h | 10 +- extensions/jni/jvm/JniProcessSession.cpp | 5 +- extensions/jni/jvm/JniProcessSession.h | 8 +- extensions/jni/jvm/JniReferenceObjects.h | 20 +-- extensions/jni/jvm/NarClassLoader.h | 20 +-- extensions/rocksdb-repos/CMakeLists.txt | 3 +- .../rocksdb-repos/DatabaseContentRepository.cpp | 2 +- .../rocksdb-repos/DatabaseContentRepository.h | 13 +- extensions/rocksdb-repos/FlowFileRepository.cpp | 3 +- extensions/rocksdb-repos/FlowFileRepository.h | 17 ++- extensions/rocksdb-repos/ProvenanceRepository.h | 22 ++-- extensions/rocksdb-repos/RocksDBLoader.h | 9 +- extensions/rocksdb-repos/RocksDbStream.h | 10 +- .../RocksDbPersistableKeyValueStoreService.cpp | 4 +- .../RocksDbPersistableKeyValueStoreService.h | 18 +-- extensions/rocksdb-repos/database/ColumnHandle.h | 1 + extensions/rocksdb-repos/database/OpenRocksDb.cpp | 2 + extensions/rocksdb-repos/database/OpenRocksDb.h | 3 + .../rocksdb-repos/database/RocksDatabase.cpp | 4 + extensions/rocksdb-repos/database/RocksDatabase.h | 8 +- .../rocksdb-repos/database/RocksDbInstance.cpp | 1 + .../rocksdb-repos/database/RocksDbInstance.h | 2 + extensions/rocksdb-repos/database/RocksDbUtils.h | 1 + extensions/rocksdb-repos/database/StringAppender.h | 8 +- extensions/script/CMakeLists.txt | 2 +- extensions/script/ExecuteScript.h | 14 +- extensions/script/ScriptEngine.h | 5 +- extensions/script/ScriptException.h | 13 +- extensions/script/ScriptFlowFile.cpp | 2 +- extensions/script/ScriptFlowFile.h | 10 +- extensions/script/ScriptProcessContext.cpp | 1 + extensions/script/ScriptProcessContext.h | 10 +- extensions/script/lua/LuaBaseStream.h | 11 +- extensions/script/lua/LuaProcessSession.cpp | 1 + extensions/script/lua/LuaProcessSession.h | 12 +- extensions/script/lua/LuaScriptEngine.h | 13 +- extensions/script/python/PyBaseStream.h | 10 +- extensions/script/python/PyProcCreator.h | 24 +--- extensions/script/python/PyProcessSession.cpp | 3 +- extensions/script/python/PyProcessSession.h | 14 +- extensions/script/python/PythonBindings.h | 11 +- extensions/script/python/PythonCreator.h | 11 +- extensions/script/python/PythonProcessor.cpp | 3 +- extensions/script/python/PythonProcessor.h | 13 +- extensions/script/python/PythonScriptEngine.h | 17 +-- extensions/script/pythonloader/PyProcLoader.h | 10 +- extensions/sensors/CMakeLists.txt | 1 + extensions/sensors/GetEnvironmentalSensors.h | 11 +- extensions/sensors/GetMovementSensors.h | 10 +- extensions/sensors/SensorBase.h | 31 ++--- extensions/sensors/SensorLoader.h | 11 +- extensions/sftp/CMakeLists.txt | 1 + extensions/sftp/SFTPLoader.h | 8 +- extensions/sftp/client/SFTPClient.cpp | 16 ++- extensions/sftp/client/SFTPClient.h | 18 +-- extensions/sftp/processors/FetchSFTP.cpp | 77 ++++++----- extensions/sftp/processors/FetchSFTP.h | 8 +- extensions/sftp/processors/ListSFTP.cpp | 138 +++++++++++--------- extensions/sftp/processors/ListSFTP.h | 16 +-- extensions/sftp/processors/PutSFTP.cpp | 143 +++++++++++---------- extensions/sftp/processors/PutSFTP.h | 16 +-- extensions/sftp/processors/SFTPProcessorBase.cpp | 121 +++++++++-------- extensions/sftp/processors/SFTPProcessorBase.h | 10 +- extensions/sftp/tests/FetchSFTPTests.cpp | 2 +- extensions/sftp/tests/ListSFTPTests.cpp | 6 +- extensions/sftp/tests/PutSFTPTests.cpp | 7 +- extensions/sftp/tests/tools/SFTPTestServer.cpp | 20 ++- extensions/sftp/tests/tools/SFTPTestServer.h | 15 +-- 89 files changed, 654 insertions(+), 666 deletions(-) diff --git a/extensions/jni/CMakeLists.txt b/extensions/jni/CMakeLists.txt index 4e9ee72..081d317 100644 --- a/extensions/jni/CMakeLists.txt +++ b/extensions/jni/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) +include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) message(STATUS "JAVA_HOME: '$ENV{JAVA_HOME}'") find_package(JNI REQUIRED) @@ -26,7 +26,7 @@ find_package(Maven REQUIRED) message(STATUS "MAVEN: ${MAVEN_EXECUTABLE}") include_directories(${JNI_INCLUDE_DIRS}) -file(GLOB SOURCES "jvm/*.cpp" "*.cpp") +file(GLOB SOURCES "jvm/*.cpp" "*.cpp") add_library(minifi-jni STATIC ${SOURCES}) set_property(TARGET minifi-jni PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -61,11 +61,11 @@ execute_process(COMMAND "${MAVEN_EXECUTABLE}" "-q" "package" "-Denforcer.skip=tr endif() if("${mvn_result}" STREQUAL "0") message("${mvn_output}") - + install(DIRECTORY "${JNI_ASSEMBLY_JAR_BIN}/nifi-minifi-jni-assembly/target/nifi-minifi-jni-minifi-jni/nifi-minifi-jni-1.9.0/minifi-jni" DESTINATION . COMPONENT bin) - + else() message("Maven failed ${mvn_result} ${mvn_output} ${mvn_error}") endif() @@ -92,7 +92,7 @@ if("${mvn_result}" STREQUAL "0") message("${mvn_output}") install(FILES ${JNI-FRAMEWORK-JAR} DESTINATION minifi-jni/lib - COMPONENT bin) + COMPONENT bin) else() message("Maven could not be invoked to build the framework jar") endif() @@ -112,4 +112,4 @@ endif() SET (JNI-EXTENSION minifi-jni PARENT_SCOPE) register_extension(minifi-jni) - +register_extension_linter(minifi-jni-linter) diff --git a/extensions/jni/ClassRegistrar.h b/extensions/jni/ClassRegistrar.h index 72c5a75..8afdeea 100644 --- a/extensions/jni/ClassRegistrar.h +++ b/extensions/jni/ClassRegistrar.h @@ -16,10 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNI_CLASSREGISTRAR_H_ -#define EXTENSIONS_JNI_CLASSREGISTRAR_H_ +#pragma once #include <memory> +#include <string> +#include <set> #include "FlowFileRecord.h" #include "core/Processor.h" @@ -75,5 +76,3 @@ class ClassRegistrar { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNI_CLASSREGISTRAR_H_ */ diff --git a/extensions/jni/ExecuteJavaControllerService.h b/extensions/jni/ExecuteJavaControllerService.h index e7be54c2c..5493f79 100644 --- a/extensions/jni/ExecuteJavaControllerService.h +++ b/extensions/jni/ExecuteJavaControllerService.h @@ -16,11 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __EXECUTE_JAVA_CS_ -#define __EXECUTE_JAVA_CS_ +#pragma once #include <memory> #include <regex> +#include <string> #include "FlowFileRecord.h" #include "core/controller/ControllerService.h" @@ -55,7 +55,6 @@ namespace controllers { */ class ExecuteJavaControllerService : public ConfigurationContext, public std::enable_shared_from_this<ConfigurationContext> { public: - // Constructor /*! * Create a new processor @@ -136,10 +135,7 @@ class ExecuteJavaControllerService : public ConfigurationContext, public std::en return methodSignatures; } - protected: - private: - minifi::jni::JniConfigurationContext config_context_; jobject contextInstance; @@ -163,5 +159,3 @@ REGISTER_RESOURCE(ExecuteJavaControllerService, "ExecuteJavaClass runs NiFi Cont } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/jni/ExecuteJavaProcessor.cpp b/extensions/jni/ExecuteJavaProcessor.cpp index 27ac3ec..e7b28ae 100644 --- a/extensions/jni/ExecuteJavaProcessor.cpp +++ b/extensions/jni/ExecuteJavaProcessor.cpp @@ -49,10 +49,13 @@ namespace minifi { namespace jni { namespace processors { -core::Property ExecuteJavaProcessor::JVMControllerService( - core::PropertyBuilder::createProperty("JVM Controller Service")->withDescription("Name of controller service defined within this flow")->isRequired(false)->withDefaultValue<std::string>("")->build()); -core::Property ExecuteJavaProcessor::NiFiProcessor( - core::PropertyBuilder::createProperty("NiFi Processor")->withDescription("Name of NiFi processor to load and run")->isRequired(true)->withDefaultValue<std::string>("")->build()); +core::Property ExecuteJavaProcessor::JVMControllerService(core::PropertyBuilder::createProperty("JVM Controller Service") + ->withDescription("Name of controller service defined within this flow") + ->isRequired(false)->withDefaultValue<std::string>("")->build()); + +core::Property ExecuteJavaProcessor::NiFiProcessor(core::PropertyBuilder::createProperty("NiFi Processor") + ->withDescription("Name of NiFi processor to load and run") + ->isRequired(true)->withDefaultValue<std::string>("")->build()); const char *ExecuteJavaProcessor::ProcessorName = "ExecuteJavaClass"; diff --git a/extensions/jni/ExecuteJavaProcessor.h b/extensions/jni/ExecuteJavaProcessor.h index 8b01511..f835ee7 100644 --- a/extensions/jni/ExecuteJavaProcessor.h +++ b/extensions/jni/ExecuteJavaProcessor.h @@ -16,10 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __EXECUTE_JAVA_CLASS__ -#define __EXECUTE_JAVA_CLASS__ +#pragma once #include <memory> +#include <vector> +#include <string> #include "FlowFileRecord.h" #include "core/Processor.h" @@ -52,7 +53,6 @@ namespace processors { */ class ExecuteJavaProcessor : public core::Processor { public: - // Constructor /*! * Create a new processor @@ -82,7 +82,6 @@ class ExecuteJavaProcessor : public core::Processor { } protected: - static JavaSignatures &getLoggerSignatures() { static JavaSignatures loggersignatures; if (loggersignatures.empty()) { @@ -104,12 +103,21 @@ class ExecuteJavaProcessor : public core::Processor { static JavaSignatures &getProcessContextSignatures() { static JavaSignatures methodSignatures; if (methodSignatures.empty()) { - methodSignatures.addSignature({ "getComponent", "()Lorg/apache/nifi/components/AbstractConfigurableComponent;", reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getComponent) }); - methodSignatures.addSignature({ "getPropertyNames", "()Ljava/util/List;", reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getPropertyNames) }); - methodSignatures.addSignature({ "getPropertyValue", "(Ljava/lang/String;)Ljava/lang/String;", reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getPropertyValue) }); - methodSignatures.addSignature({ "getName", "()Ljava/lang/String;", reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getName) }); - methodSignatures.addSignature({ "getControllerServiceLookup", "()Lorg/apache/nifi/controller/ControllerServiceLookup;", - reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getControllerServiceLookup) }); + methodSignatures.addSignature({ "getComponent", + "()Lorg/apache/nifi/components/AbstractConfigurableComponent;", + reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getComponent) }); + methodSignatures.addSignature({ "getPropertyNames", + "()Ljava/util/List;", + reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getPropertyNames) }); + methodSignatures.addSignature({ "getPropertyValue", + "(Ljava/lang/String;)Ljava/lang/String;", + reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getPropertyValue) }); + methodSignatures.addSignature({ "getName", + "()Ljava/lang/String;", + reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getName) }); + methodSignatures.addSignature({ "getControllerServiceLookup", + "()Lorg/apache/nifi/controller/ControllerServiceLookup;", + reinterpret_cast<void*>(&Java_org_apache_nifi_processor_JniProcessContext_getControllerServiceLookup) }); } return methodSignatures; } @@ -234,7 +242,6 @@ class ExecuteJavaProcessor : public core::Processor { } private: - JniSessionFactory *getFactory(const std::shared_ptr<core::ProcessSessionFactory> &ptr) { std::lock_guard<std::mutex> lock(local_mutex_); for (const auto &factory : session_factories_) { @@ -297,5 +304,3 @@ REGISTER_RESOURCE(ExecuteJavaProcessor, "ExecuteJavaClass runs NiFi processors g } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/jni/JNILoader.h b/extensions/jni/JNILoader.h index 86467d0..6a0dc03 100644 --- a/extensions/jni/JNILoader.h +++ b/extensions/jni/JNILoader.h @@ -15,8 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNILOADER_H -#define EXTENSIONS_JNILOADER_H +#pragma once + +#include <vector> +#include <string> +#include <memory> #include "core/ClassLoader.h" #include "ExecuteJavaProcessor.h" @@ -70,10 +73,8 @@ class JNIFactory : public core::ObjectFactory { static minifi::jni::JVMLoader jvm; static bool added; -} -; +}; extern "C" { DLL_EXPORT void *createJNIFactory(void); } -#endif /* EXTENSIONS_JNILOADER_H */ diff --git a/extensions/jni/JNIUtil.h b/extensions/jni/JNIUtil.h index f3277fc..91fc163 100644 --- a/extensions/jni/JNIUtil.h +++ b/extensions/jni/JNIUtil.h @@ -16,11 +16,10 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_JNIUTIL_H -#define NIFI_MINIFI_CPP_JNIUTIL_H +#pragma once -#include <string> #include <jni.h> +#include <string> static inline std::string JniStringToUTF(JNIEnv *env, const jstring &jstr) { if (!jstr && !env) { @@ -34,5 +33,3 @@ static inline std::string JniStringToUTF(JNIEnv *env, const jstring &jstr) { env->ReleaseStringUTFChars(jstr, c_str); return str; } - -#endif // NIFI_MINIFI_CPP_JNIUTIL_H diff --git a/extensions/jni/JVMCreator.h b/extensions/jni/JVMCreator.h index e4f9d83..c64dcfa 100644 --- a/extensions/jni/JVMCreator.h +++ b/extensions/jni/JVMCreator.h @@ -16,8 +16,7 @@ * limitations under the License. */ -#ifndef EXTENSIONS_JNI_JVMCREATOR_H_ -#define EXTENSIONS_JNI_JVMCREATOR_H_ +#pragma once #include <vector> #include <string> @@ -39,7 +38,6 @@ namespace jni { */ class JVMCreator : public minifi::core::CoreComponent { public: - explicit JVMCreator(const std::string &name, const utils::Identifier &uuid = {}) : minifi::core::CoreComponent(name, uuid), loader_(nullptr), @@ -94,7 +92,6 @@ class JVMCreator : public minifi::core::CoreComponent { } private: - minifi::jni::JVMLoader *loader_; std::vector<std::string> jvm_options_; @@ -109,5 +106,3 @@ class JVMCreator : public minifi::core::CoreComponent { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNI_JVMCREATOR_H_ */ diff --git a/extensions/jni/JavaException.h b/extensions/jni/JavaException.h index 271bfcf..3c35bfe 100644 --- a/extensions/jni/JavaException.h +++ b/extensions/jni/JavaException.h @@ -17,15 +17,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __JAVA_EXCEPTION_H__ -#define __JAVA_EXCEPTION_H__ +#pragma once + +#include <errno.h> +#include <jni.h> #include <sstream> #include <stdexcept> -#include <errno.h> -#include <string.h> +#include <string> +#include <utility> + #include "core/expect.h" -#include <jni.h> #include "jvm/JavaDefs.h" #include "JNIUtil.h" @@ -44,7 +46,7 @@ class JavaException : public std::exception { /*! * Create a new JavaException */ - JavaException(std::string errorMsg) + explicit JavaException(std::string errorMsg) : message_(std::move(errorMsg)) { } @@ -113,5 +115,3 @@ static inline void ThrowJava(JNIEnv *env, const char *message) { #define THROW_IF_NULL(expr, env, message) if (UNLIKELY(expr == nullptr)) minifi::jni::ThrowJava(env, message) #define THROW_IF(expr, env, message) if (UNLIKELY(expr)) minifi::jni::ThrowJava(env, message) - -#endif diff --git a/extensions/jni/jvm/JVMLoader.h b/extensions/jni/jvm/JVMLoader.h index 11560f2..cbe02e9 100644 --- a/extensions/jni/jvm/JVMLoader.h +++ b/extensions/jni/jvm/JVMLoader.h @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JVMLOADER_H -#define EXTENSIONS_JVMLOADER_H +#pragma once +#include <jni.h> +#include <memory> +#include <utility> #include <string> #include <map> #include <vector> @@ -28,7 +30,7 @@ #include "JavaServicer.h" #include "../JavaException.h" #include "core/Core.h" -#include <jni.h> + #ifndef WIN32 #include <dlfcn.h> #endif @@ -88,7 +90,6 @@ void setPtr(JNIEnv *env, jobject obj, T *t); */ class JVMLoader { public: - bool initialized() { return initialized_; } @@ -99,10 +100,10 @@ class JVMLoader { */ JNIEnv *attach(const std::string& /*name*/ = "") { JNIEnv* jenv; - jint ret = jvm_->GetEnv((void**) &jenv, JNI_VERSION_1_8); + jint ret = jvm_->GetEnv(reinterpret_cast<void**>(&jenv), JNI_VERSION_1_8); if (ret == JNI_EDETACHED) { - ret = jvm_->AttachCurrentThread((void**) &jenv, NULL); + ret = jvm_->AttachCurrentThread(reinterpret_cast<void**>(&jenv), NULL); if (ret != JNI_OK || jenv == NULL) { throw std::runtime_error("Could not find class"); } @@ -220,8 +221,9 @@ class JVMLoader { #else str << ":" << path; #endif - } else + } else { str << path; + } } options.insert(options.end(), otherOptions.begin(), otherOptions.end()); std::string classpath = "-Djava.class.path=" + str.str(); @@ -294,7 +296,6 @@ class JVMLoader { } protected: - static FieldMapping &getClassMapping() { static FieldMapping map; return map; @@ -348,7 +349,7 @@ class JVMLoader { #ifdef _MSC_VER #pragma warning(suppress: 4054) #endif - return (void*)symbol; + return reinterpret_cast<void*>(symbol); } const char *dlerror(void) { @@ -383,8 +384,7 @@ class JVMLoader { resource_mapping_.insert(std::make_pair(allModules[i], "minifi-system")); } } - } - else { + } else { char lpFileName[MAX_PATH]; int i; @@ -407,7 +407,7 @@ class JVMLoader { /* Return to previous state of the error-mode bit flags. */ SetErrorMode(uMode); - return (void *)object; + return reinterpret_cast<void *>(object); } int dlclose(void *handle) { @@ -423,7 +423,7 @@ class JVMLoader { ret = !ret; - return (int)ret; + return static_cast<int>(ret); } #endif @@ -456,7 +456,7 @@ class JVMLoader { vm_args.ignoreUnrecognized = JNI_FALSE; // load and initialize a Java VM, return a JNI interface // pointer in env - JNI_CreateJavaVM(&jvm_, (void**) &env_, &vm_args); + JNI_CreateJavaVM(&jvm_, reinterpret_cast<void **>(&env_), &vm_args); // we're actually using a known class to locate the class loader and provide it // to referentially perform lookups. auto randomClass = find_class_global(env_, "org/apache/nifi/processor/ProcessContext"); @@ -472,7 +472,6 @@ class JVMLoader { } private: - std::atomic<bool> initialized_; std::map<std::string, JavaClass> objects_; @@ -511,5 +510,3 @@ class JVMLoader { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JVMLOADER_H */ diff --git a/extensions/jni/jvm/JavaClass.h b/extensions/jni/jvm/JavaClass.h index 6797447..e3d5eeb 100644 --- a/extensions/jni/jvm/JavaClass.h +++ b/extensions/jni/jvm/JavaClass.h @@ -15,15 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JAVACLASS_H -#define EXTENSIONS_JAVACLASS_H +#pragma once + +#include <jni.h> #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> +#include <utility> + #include "JniProcessContext.h" #include "JniFlowFile.h" #include "JniProcessSession.h" @@ -43,7 +45,6 @@ namespace jni { */ class JavaClass { public: - JavaClass() : class_ref_(nullptr) { } @@ -130,5 +131,3 @@ class JavaClass { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JAVACLASS_H */ diff --git a/extensions/jni/jvm/JavaControllerService.cpp b/extensions/jni/jvm/JavaControllerService.cpp index b5f5911..1686c20 100644 --- a/extensions/jni/jvm/JavaControllerService.cpp +++ b/extensions/jni/jvm/JavaControllerService.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "../jvm/JavaControllerService.h" +#include "JavaControllerService.h" #include <string> #include <memory> diff --git a/extensions/jni/jvm/JavaControllerService.h b/extensions/jni/jvm/JavaControllerService.h index 44add0e..42f92bc 100644 --- a/extensions/jni/jvm/JavaControllerService.h +++ b/extensions/jni/jvm/JavaControllerService.h @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_CONTROLLERS_JAVACONTROLLERSERVICE_H_ -#define LIBMINIFI_INCLUDE_CONTROLLERS_JAVACONTROLLERSERVICE_H_ +#pragma once #include <iostream> #include <memory> #include <vector> #include <string> +#include <utility> +#include <map> #include "../jvm/JVMLoader.h" #include "NarClassLoader.h" @@ -143,12 +144,7 @@ class JavaControllerService : public core::controller::ControllerService, public return nar_loader_->newInstance(requested_name); } - protected: - - // void addPath(std::vector<std::string> &jarFiles, const std::string &dir); - private: - JavaClass narClassLoaderClazz; std::mutex initialization_mutex_; @@ -172,5 +168,3 @@ REGISTER_RESOURCE(JavaControllerService, "Allows specification of nars to be use } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* LIBMINIFI_INCLUDE_CONTROLLERS_JAVACONTROLLERSERVICE_H_ */ diff --git a/extensions/jni/jvm/JavaServicer.h b/extensions/jni/jvm/JavaServicer.h index 580d997..95c48a8 100644 --- a/extensions/jni/jvm/JavaServicer.h +++ b/extensions/jni/jvm/JavaServicer.h @@ -20,6 +20,7 @@ #define EXTENSIONS_JNI_JVM_JAVASERVICER_H_ #include <jni.h> +#include <string> #include "JavaClass.h" namespace org { diff --git a/extensions/jni/jvm/JniBundle.h b/extensions/jni/jvm/JniBundle.h index 5521993..d3e3994 100644 --- a/extensions/jni/jvm/JniBundle.h +++ b/extensions/jni/jvm/JniBundle.h @@ -15,15 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNIBUNDLE_H -#define EXTENSIONS_JNIBUNDLE_H +#pragma once + +#include <jni.h> #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + #include "JniProcessContext.h" #include "JniFlowFile.h" #include "JniProcessSession.h" @@ -41,7 +42,6 @@ namespace jni { */ class JniBundle { public: - explicit JniBundle(struct BundleDetails details) : details_(details) { } @@ -80,5 +80,3 @@ class JniBundle { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNIBUNDLE_H */ diff --git a/extensions/jni/jvm/JniConfigurationContext.h b/extensions/jni/jvm/JniConfigurationContext.h index f5a0e53..add88e9 100644 --- a/extensions/jni/jvm/JniConfigurationContext.h +++ b/extensions/jni/jvm/JniConfigurationContext.h @@ -16,8 +16,10 @@ * limitations under the License. */ -#ifndef EXTENSIONS_JNI_JVM_JNICONFIGURATIONCONTEXT_H_ -#define EXTENSIONS_JNI_JVM_JNICONFIGURATIONCONTEXT_H_ +#pragma once + +#include <string> +#include <memory> #include "core/controller/ControllerService.h" @@ -29,7 +31,6 @@ namespace jni { class ConfigurationContext : public core::controller::ControllerService { public: - // Constructor /*! * Create a new processor @@ -51,5 +52,3 @@ struct JniConfigurationContext { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNI_JVM_JNICONFIGURATIONCONTEXT_H_ */ diff --git a/extensions/jni/jvm/JniControllerServiceLookup.h b/extensions/jni/jvm/JniControllerServiceLookup.h index 5dd1294..43a2a8b 100644 --- a/extensions/jni/jvm/JniControllerServiceLookup.h +++ b/extensions/jni/jvm/JniControllerServiceLookup.h @@ -19,12 +19,15 @@ #ifndef EXTENSIONS_JNI_JVM_JNICONTROLLERSERVICELOOKUP_H_ #define EXTENSIONS_JNI_JVM_JNICONTROLLERSERVICELOOKUP_H_ +#include <jni.h> + #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> +#include <memory> + #include "core/Processor.h" #include "core/ProcessSession.h" @@ -36,7 +39,6 @@ namespace jni { class JniControllerService { public: - std::shared_ptr<core::controller::ControllerService> cs_reference_; }; diff --git a/extensions/jni/jvm/JniFlowFile.h b/extensions/jni/jvm/JniFlowFile.h index 88bb1e6..54a9e40 100644 --- a/extensions/jni/jvm/JniFlowFile.h +++ b/extensions/jni/jvm/JniFlowFile.h @@ -19,12 +19,14 @@ #ifndef EXTENSIONS_JNI_JVM_JNIFLOWFILE_H_ #define EXTENSIONS_JNI_JVM_JNIFLOWFILE_H_ +#include <jni.h> + #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + #include "core/Processor.h" #include "core/ProcessSession.h" diff --git a/extensions/jni/jvm/JniInitializationContext.h b/extensions/jni/jvm/JniInitializationContext.h index aae970d..13843ce 100644 --- a/extensions/jni/jvm/JniInitializationContext.h +++ b/extensions/jni/jvm/JniInitializationContext.h @@ -15,15 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNIINITIALIZATIONCONTEXT_H -#define EXTENSIONS_JNIINITIALIZATIONCONTEXT_H +#pragma once + +#include <jni.h> #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + #include "core/Processor.h" #include "core/ProcessSession.h" #include "JniControllerServiceLookup.h" @@ -72,6 +73,3 @@ JNIEXPORT jstring JNICALL Java_org_apache_nifi_processor_JniConfigurationContext #ifdef __cplusplus } #endif - - -#endif /* EXTENSIONS_JNIINITIALIZATIONCONTEXT_H */ diff --git a/extensions/jni/jvm/JniLogger.h b/extensions/jni/jvm/JniLogger.h index e65689c..f2dd452 100644 --- a/extensions/jni/jvm/JniLogger.h +++ b/extensions/jni/jvm/JniLogger.h @@ -15,8 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNILOGGER_H -#define EXTENSIONS_JNILOGGER_H +#pragma once + +#include <jni.h> #include <string> #include <memory> @@ -24,7 +25,7 @@ #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + #include "core/logging/LoggerConfiguration.h" namespace org { @@ -67,5 +68,3 @@ JNIEXPORT void JNICALL Java_org_apache_nifi_processor_JniLogger_trace(JNIEnv *en #ifdef __cplusplus } #endif - -#endif /* EXTENSIONS_JNILOGGER_H */ diff --git a/extensions/jni/jvm/JniMethod.h b/extensions/jni/jvm/JniMethod.h index 1042cc9..3ac899a 100644 --- a/extensions/jni/jvm/JniMethod.h +++ b/extensions/jni/jvm/JniMethod.h @@ -15,15 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNI_JVM_JNIMETHOD_H_ -#define EXTENSIONS_JNI_JVM_JNIMETHOD_H_ +#pragma once +#include <jni.h> + +#include <memory> #include <string> +#include <utility> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + namespace org { namespace apache { @@ -72,7 +75,6 @@ class JavaMethodSignature { JavaMethodSignature &operator=(JavaMethodSignature &&other) = default; private: - std::string name_; std::string params_; void *ptr_; @@ -112,6 +114,7 @@ class JavaSignatures { size_t getSize() const { return size_; } + private: mutable std::mutex mutex_; mutable std::unique_ptr<JNINativeMethod[]> method_ptr_; @@ -124,5 +127,3 @@ class JavaSignatures { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNI_JVM_JNIMETHOD_H_ */ diff --git a/extensions/jni/jvm/JniProcessContext.h b/extensions/jni/jvm/JniProcessContext.h index 1196d7b..17d45ed 100644 --- a/extensions/jni/jvm/JniProcessContext.h +++ b/extensions/jni/jvm/JniProcessContext.h @@ -15,15 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNIPROCESSCONTEXT_H -#define EXTENSIONS_JNIPROCESSCONTEXT_H +#pragma once + +#include <jni.h> #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> +#include <memory> + #include "core/Processor.h" #include "core/ProcessSession.h" #include "JniControllerServiceLookup.h" @@ -72,5 +74,3 @@ DLL_EXPORT jobject JNICALL Java_org_apache_nifi_processor_JniProcessContext_getC #ifdef __cplusplus } #endif - -#endif /* EXTENSIONS_JNIPROCESSCONTEXT_H */ diff --git a/extensions/jni/jvm/JniProcessSession.cpp b/extensions/jni/jvm/JniProcessSession.cpp index 56be969..b8ec029 100644 --- a/extensions/jni/jvm/JniProcessSession.cpp +++ b/extensions/jni/jvm/JniProcessSession.cpp @@ -23,6 +23,7 @@ #include <algorithm> #include <iterator> #include <set> +#include <utility> #include "core/Property.h" #include "io/validation.h" #include "utils/StringUtils.h" @@ -113,7 +114,7 @@ JNIEXPORT jint JNICALL Java_org_apache_nifi_processor_JniInputStream_readWithOf // this technically can't happen per JNI specs return -1; } - return jin->read(env, arr, (int) offset, (int) length); + return jin->read(env, arr, static_cast<int>(offset), static_cast<int>(length)); } JNIEXPORT jboolean JNICALL Java_org_apache_nifi_processor_JniProcessSession_write(JNIEnv *env, jobject obj, jobject ff, jbyteArray byteArray) { @@ -254,7 +255,7 @@ jstring Java_org_apache_nifi_processor_JniProcessSession_getPropertyValue(JNIEnv std::string keystr = JniStringToUTF(env, propertyName); if (!context->getProperty(keystr, value)) { context->getDynamicProperty(keystr, value); - }; + } return env->NewStringUTF(value.c_str()); } diff --git a/extensions/jni/jvm/JniProcessSession.h b/extensions/jni/jvm/JniProcessSession.h index ddec7ea..8bd2864 100644 --- a/extensions/jni/jvm/JniProcessSession.h +++ b/extensions/jni/jvm/JniProcessSession.h @@ -15,16 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_JNIPROCESSSESSION_H -#define EXTENSIONS_JNIPROCESSSESSION_H +#pragma once +#include <jni.h> #include <string> #include <memory> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> + #include "io/BaseStream.h" #include "FlowFileRecord.h" #include "core/ProcessSession.h" @@ -86,5 +86,3 @@ JNIEXPORT jint JNICALL Java_org_apache_nifi_processor_JniInputStream_readWithOff #ifdef __cplusplus } #endif - -#endif /* EXTENSIONS_JNIPROCESSSESSION_H */ diff --git a/extensions/jni/jvm/JniReferenceObjects.h b/extensions/jni/jvm/JniReferenceObjects.h index b7ef344..83d7c89 100644 --- a/extensions/jni/jvm/JniReferenceObjects.h +++ b/extensions/jni/jvm/JniReferenceObjects.h @@ -16,15 +16,19 @@ * limitations under the License. */ -#ifndef EXTENSIONS_JNI_JVM_REFERNCEOBJECTS_H_ -#define EXTENSIONS_JNI_JVM_REFERNCEOBJECTS_H_ +#pragma once + +#include <jni.h> #include <string> #include <vector> #include <sstream> #include <iterator> #include <algorithm> -#include <jni.h> +#include <functional> +#include <memory> +#include <utility> + #include "JavaServicer.h" #include "core/Processor.h" #include "core/ProcessSession.h" @@ -72,7 +76,6 @@ class JniFlowFile : public core::WeakReference { } protected: - bool removed; jobject ff_object; @@ -102,7 +105,7 @@ class JniByteOutStream : public minifi::OutputStreamCallback { virtual ~JniByteOutStream() = default; virtual int64_t process(const std::shared_ptr<minifi::io::BaseStream>& stream) { - const auto write_ret = stream->write((uint8_t*)bytes_, length_); + const auto write_ret = stream->write(reinterpret_cast<uint8_t*>(bytes_), length_); return io::isError(write_ret) ? -1 : gsl::narrow<int64_t>(write_ret); } private: @@ -115,7 +118,7 @@ class JniByteOutStream : public minifi::OutputStreamCallback { */ class JniByteInputStream : public minifi::InputStreamCallback { public: - JniByteInputStream(uint64_t size) + explicit JniByteInputStream(uint64_t size) : stream_(nullptr), read_size_(0) { buffer_size_ = size; @@ -152,7 +155,7 @@ class JniByteInputStream : public minifi::InputStreamCallback { } read += actual; - env->SetByteArrayRegion(arr, offset + writtenOffset, actual, (jbyte*) buffer_); + env->SetByteArrayRegion(arr, offset + writtenOffset, actual, reinterpret_cast<jbyte*>(buffer_)); writtenOffset += actual; remaining -= actual; @@ -299,7 +302,6 @@ struct check_empty : public std::unary_function<std::shared_ptr<JniSession>, boo class JniSessionFactory : public core::WeakReference { public: - JniSessionFactory(const std::shared_ptr<core::ProcessSessionFactory> &factory, const std::shared_ptr<JavaServicer> &servicer, jobject java_object) : servicer_(servicer), factory_(factory), @@ -363,5 +365,3 @@ class JniSessionFactory : public core::WeakReference { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_JNI_JVM_REFERNCEOBJECTS_H_ */ diff --git a/extensions/jni/jvm/NarClassLoader.h b/extensions/jni/jvm/NarClassLoader.h index 706b3a6..5173767 100644 --- a/extensions/jni/jvm/NarClassLoader.h +++ b/extensions/jni/jvm/NarClassLoader.h @@ -15,18 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_NARCLASSLOADER_H -#define EXTENSIONS_NARCLASSLOADER_H +#pragma once + +#include <jni.h> -#include "JavaServicer.h" -#include "JniBundle.h" -#include "../JavaException.h" #include <string> #include <vector> #include <sstream> #include <iterator> +#include <memory> +#include <utility> +#include <map> #include <algorithm> -#include <jni.h> + +#include "JavaServicer.h" +#include "JniBundle.h" +#include "../JavaException.h" namespace org { namespace apache { @@ -36,7 +40,6 @@ namespace jni { class NarClassLoader { public: - NarClassLoader(std::shared_ptr<minifi::jni::JavaServicer> servicer, JavaClass &clazz, const std::string &dir_name, const std::string &scratch_nar_dir, const std::string &docs_dir) : java_servicer_(servicer) { class_ref_ = clazz; @@ -196,7 +199,6 @@ class NarClassLoader { } private: - /** * Call empty constructor */ @@ -469,5 +471,3 @@ class NarClassLoader { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_NARCLASSLOADER_H */ diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt index 67a1e52..f7f97d5 100644 --- a/extensions/rocksdb-repos/CMakeLists.txt +++ b/extensions/rocksdb-repos/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) +include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) file(GLOB SOURCES "*.cpp" "controllers/*.cpp" "database/*.cpp") @@ -29,3 +29,4 @@ target_link_libraries(minifi-rocksdb-repos RocksDB::RocksDB) SET (ROCKSDB-REPOS minifi-rocksdb-repos PARENT_SCOPE) register_extension(minifi-rocksdb-repos) +register_extension_linter(minifi-rocksdb-repos-linter) diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.cpp b/extensions/rocksdb-repos/DatabaseContentRepository.cpp index 19075be..dc78af7 100644 --- a/extensions/rocksdb-repos/DatabaseContentRepository.cpp +++ b/extensions/rocksdb-repos/DatabaseContentRepository.cpp @@ -20,9 +20,9 @@ #include <memory> #include <string> +#include <utility> #include "RocksDbStream.h" -#include "rocksdb/merge_operator.h" #include "utils/GeneralUtils.h" #include "utils/gsl.h" #include "Exception.h" diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.h b/extensions/rocksdb-repos/DatabaseContentRepository.h index 145a286..d4419d4 100644 --- a/extensions/rocksdb-repos/DatabaseContentRepository.h +++ b/extensions/rocksdb-repos/DatabaseContentRepository.h @@ -15,8 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_ -#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_ +#pragma once + +#include <string> +#include <memory> #include "core/Core.h" #include "core/Connectable.h" @@ -33,6 +35,7 @@ namespace minifi { namespace core { namespace repository { + /** * DatabaseContentRepository is a content repository that stores data onto the local file system. */ @@ -43,9 +46,9 @@ class DatabaseContentRepository : public core::ContentRepository, public core::C void commit() override; }; - public: - DatabaseContentRepository(const std::string& name = getClassName<DatabaseContentRepository>(), const utils::Identifier& uuid = {}) + public: + explicit DatabaseContentRepository(const std::string& name = getClassName<DatabaseContentRepository>(), const utils::Identifier& uuid = {}) : core::Connectable(name, uuid), is_valid_(false), db_(nullptr), @@ -105,5 +108,3 @@ class DatabaseContentRepository : public core::ContentRepository, public core::C } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_ */ diff --git a/extensions/rocksdb-repos/FlowFileRepository.cpp b/extensions/rocksdb-repos/FlowFileRepository.cpp index 0349255..8e47e3a 100644 --- a/extensions/rocksdb-repos/FlowFileRepository.cpp +++ b/extensions/rocksdb-repos/FlowFileRepository.cpp @@ -229,8 +229,9 @@ void FlowFileRepository::initialize_repository() { } else { logger_->log_trace("Could not create checkpoint. Corrupt?"); } - } else + } else { logger_->log_trace("Could not create checkpoint directory. Not properly deleted?"); + } } void FlowFileRepository::loadComponent(const std::shared_ptr<core::ContentRepository> &content_repo) { diff --git a/extensions/rocksdb-repos/FlowFileRepository.h b/extensions/rocksdb-repos/FlowFileRepository.h index ff3b36b..b7ed458 100644 --- a/extensions/rocksdb-repos/FlowFileRepository.h +++ b/extensions/rocksdb-repos/FlowFileRepository.h @@ -15,8 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ -#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ +#pragma once + +#include <utility> +#include <vector> +#include <string> +#include <memory> #include "utils/file/FileUtils.h" #include "rocksdb/db.h" @@ -61,7 +65,8 @@ class FlowFileRepository : public core::Repository, public std::enable_shared_fr : FlowFileRepository(name) { } - FlowFileRepository(const std::string repo_name = "", const std::string& checkpoint_dir = FLOWFILE_CHECKPOINT_DIRECTORY, std::string directory = FLOWFILE_REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME, + FlowFileRepository(const std::string repo_name = "", const std::string& checkpoint_dir = FLOWFILE_CHECKPOINT_DIRECTORY, + std::string directory = FLOWFILE_REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME, int64_t maxPartitionBytes = MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE, uint64_t purgePeriod = FLOWFILE_REPOSITORY_PURGE_PERIOD) : core::SerializableComponent(repo_name), Repository(repo_name.length() > 0 ? repo_name : core::getClassName<FlowFileRepository>(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod), @@ -94,7 +99,8 @@ class FlowFileRepository : public core::Repository, public std::enable_shared_fr logger_->log_debug("NiFi FlowFile Max Partition Bytes %d", max_partition_bytes_); if (configure->get(Configure::nifi_flowfile_repository_max_storage_time, value)) { TimeUnit unit; - if (Property::StringToTime(value, max_partition_millis_, unit) && Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) { + if (Property::StringToTime(value, max_partition_millis_, unit)) { + Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_); } } logger_->log_debug("NiFi FlowFile Max Storage Time: [%d] ms", max_partition_millis_); @@ -192,7 +198,6 @@ class FlowFileRepository : public core::Repository, public std::enable_shared_fr } private: - bool ExecuteWithRetry(std::function<rocksdb::Status()> operation); /** @@ -225,5 +230,3 @@ class FlowFileRepository : public core::Repository, public std::enable_shared_fr } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ */ diff --git a/extensions/rocksdb-repos/ProvenanceRepository.h b/extensions/rocksdb-repos/ProvenanceRepository.h index ee47dcb..3134c74 100644 --- a/extensions/rocksdb-repos/ProvenanceRepository.h +++ b/extensions/rocksdb-repos/ProvenanceRepository.h @@ -14,8 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ -#define LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ +#pragma once + +#include <vector> +#include <string> +#include <memory> +#include <algorithm> +#include <utility> #include "rocksdb/db.h" #include "rocksdb/options.h" @@ -80,7 +85,8 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ logger_->log_debug("MiNiFi Provenance Max Partition Bytes %d", max_partition_bytes_); if (config->get(Configure::nifi_provenance_repository_max_storage_time, value)) { core::TimeUnit unit; - if (core::Property::StringToTime(value, max_partition_millis_, unit) && core::Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) { + if (core::Property::StringToTime(value, max_partition_millis_, unit)) { + core::Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_); } } logger_->log_debug("MiNiFi Provenance Max Storage Time: [%d] ms", max_partition_millis_); @@ -156,7 +162,7 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ std::string key = it->key().ToString(); if (store.size() >= max_size) break; - if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { + if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) { store.push_back(std::dynamic_pointer_cast<core::CoreComponent>(eventRead)); } } @@ -172,7 +178,7 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ break; std::shared_ptr<core::SerializableComponent> eventRead = lambda(); std::string key = it->key().ToString(); - if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { + if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) { max_size++; records.push_back(eventRead); } @@ -188,7 +194,7 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ std::string key = it->key().ToString(); if (records.size() >= (uint64_t)maxSize) break; - if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { + if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) { records.push_back(eventRead); } } @@ -201,7 +207,7 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>(); std::string key = it->key().ToString(); - if (store.at(max_size)->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { + if (store.at(max_size)->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) { max_size++; } if (store.size() >= max_size) @@ -239,5 +245,3 @@ class ProvenanceRepository : public core::Repository, public std::enable_shared_ } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif /* LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ */ - diff --git a/extensions/rocksdb-repos/RocksDBLoader.h b/extensions/rocksdb-repos/RocksDBLoader.h index f135b47..a441cac 100644 --- a/extensions/rocksdb-repos/RocksDBLoader.h +++ b/extensions/rocksdb-repos/RocksDBLoader.h @@ -15,8 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_ROCKSDBREPOS_ROCKSDBLOADER_H -#define EXTENSIONS_ROCKSDBREPOS_ROCKSDBLOADER_H +#pragma once + +#include <vector> +#include <string> +#include <algorithm> +#include <memory> #include "DatabaseContentRepository.h" #include "FlowFileRepository.h" @@ -79,4 +83,3 @@ class RocksDBFactory : public core::ObjectFactory { extern "C" { DLL_EXPORT void *createRocksDBFactory(void); } -#endif // EXTENSIONS_ROCKSDBREPOS_ROCKSDBLOADER_H diff --git a/extensions/rocksdb-repos/RocksDbStream.h b/extensions/rocksdb-repos/RocksDbStream.h index eb54716..58463bc 100644 --- a/extensions/rocksdb-repos/RocksDbStream.h +++ b/extensions/rocksdb-repos/RocksDbStream.h @@ -15,14 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_ -#define LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_ +#pragma once -#include "database/RocksDatabase.h" #include <iostream> #include <cstdint> #include <string> -#include "io/EndianCheck.h" +#include <memory> +#include "database/RocksDatabase.h" #include "io/BaseStream.h" #include "core/logging/LoggerConfiguration.h" @@ -97,7 +96,6 @@ class RocksDbStream : public io::BaseStream { size_t size_; private: - std::shared_ptr<logging::Logger> logger_; }; @@ -106,5 +104,3 @@ class RocksDbStream : public io::BaseStream { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_ */ diff --git a/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.cpp b/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.cpp index 06c5137..522ce23 100644 --- a/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.cpp +++ b/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.cpp @@ -15,11 +15,13 @@ * limitations under the License. */ +#include <fstream> +#include <set> + #include "RocksDbPersistableKeyValueStoreService.h" #include "utils/StringUtils.h" -#include <fstream> namespace org { namespace apache { diff --git a/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.h b/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.h index e07d12c..b6ba299 100644 --- a/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.h +++ b/extensions/rocksdb-repos/controllers/RocksDbPersistableKeyValueStoreService.h @@ -14,25 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_CONTROLLERS_KEYVALUE_RocksDbPersistableKeyValueStoreService_H_ -#define LIBMINIFI_INCLUDE_CONTROLLERS_KEYVALUE_RocksDbPersistableKeyValueStoreService_H_ +#pragma once + +#include <unordered_map> +#include <string> +#include <memory> #include "controllers/keyvalue/AbstractAutoPersistingKeyValueStoreService.h" #include "core/Core.h" -#include "properties/Configure.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "../database/RocksDatabase.h" -#include "rocksdb/db.h" #include "rocksdb/options.h" -#include "rocksdb/slice.h" -#include <unordered_map> -#include <string> -#include <mutex> -#include <memory> -#include <utility> + namespace org { namespace apache { @@ -83,5 +79,3 @@ REGISTER_RESOURCE(RocksDbPersistableKeyValueStoreService, "A key-value service i } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* LIBMINIFI_INCLUDE_CONTROLLERS_KEYVALUE_RocksDbPersistableKeyValueStoreService_H_ */ diff --git a/extensions/rocksdb-repos/database/ColumnHandle.h b/extensions/rocksdb-repos/database/ColumnHandle.h index 9d33d48..e73e79f 100644 --- a/extensions/rocksdb-repos/database/ColumnHandle.h +++ b/extensions/rocksdb-repos/database/ColumnHandle.h @@ -19,6 +19,7 @@ #pragma once #include <memory> +#include <utility> #include "rocksdb/db.h" namespace org { diff --git a/extensions/rocksdb-repos/database/OpenRocksDb.cpp b/extensions/rocksdb-repos/database/OpenRocksDb.cpp index 68e18e8..b74e1a7 100644 --- a/extensions/rocksdb-repos/database/OpenRocksDb.cpp +++ b/extensions/rocksdb-repos/database/OpenRocksDb.cpp @@ -16,6 +16,8 @@ * limitations under the License. */ +#include <utility> + #include "OpenRocksDb.h" #include "ColumnHandle.h" #include "RocksDbInstance.h" diff --git a/extensions/rocksdb-repos/database/OpenRocksDb.h b/extensions/rocksdb-repos/database/OpenRocksDb.h index 683c4b5..7aed4d5 100644 --- a/extensions/rocksdb-repos/database/OpenRocksDb.h +++ b/extensions/rocksdb-repos/database/OpenRocksDb.h @@ -19,7 +19,10 @@ #pragma once #include <memory> +#include <string> +#include <vector> #include "utils/gsl.h" + #include "rocksdb/db.h" #include "rocksdb/utilities/checkpoint.h" #include "WriteBatch.h" diff --git a/extensions/rocksdb-repos/database/RocksDatabase.cpp b/extensions/rocksdb-repos/database/RocksDatabase.cpp index 464f864..a83cda2 100644 --- a/extensions/rocksdb-repos/database/RocksDatabase.cpp +++ b/extensions/rocksdb-repos/database/RocksDatabase.cpp @@ -17,6 +17,10 @@ */ #include "RocksDatabase.h" + +#include <unordered_map> +#include <utility> + #include "core/logging/LoggerConfiguration.h" #include "utils/StringUtils.h" #include "RocksDbInstance.h" diff --git a/extensions/rocksdb-repos/database/RocksDatabase.h b/extensions/rocksdb-repos/database/RocksDatabase.h index 844d84f..fda0d36 100644 --- a/extensions/rocksdb-repos/database/RocksDatabase.h +++ b/extensions/rocksdb-repos/database/RocksDatabase.h @@ -18,6 +18,9 @@ #pragma once +#include <memory> +#include <string> + #include "utils/OptionalUtils.h" #include "rocksdb/db.h" #include "logging/Logger.h" @@ -38,7 +41,10 @@ class RocksDbInstance; */ class RocksDatabase { public: - static std::unique_ptr<RocksDatabase> create(const DBOptionsPatch& db_options_patch, const ColumnFamilyOptionsPatch& cf_options_patch, const std::string& uri, RocksDbMode mode = RocksDbMode::ReadWrite); + static std::unique_ptr<RocksDatabase> create(const DBOptionsPatch& db_options_patch, + const ColumnFamilyOptionsPatch& cf_options_patch, + const std::string& uri, + RocksDbMode mode = RocksDbMode::ReadWrite); RocksDatabase(std::shared_ptr<RocksDbInstance> db, std::string column, DBOptionsPatch db_options_patch, ColumnFamilyOptionsPatch cf_options_patch); diff --git a/extensions/rocksdb-repos/database/RocksDbInstance.cpp b/extensions/rocksdb-repos/database/RocksDbInstance.cpp index 6c36d8b..39bdaa6 100644 --- a/extensions/rocksdb-repos/database/RocksDbInstance.cpp +++ b/extensions/rocksdb-repos/database/RocksDbInstance.cpp @@ -17,6 +17,7 @@ */ #include "RocksDbInstance.h" +#include <vector> #include "logging/LoggerConfiguration.h" #include "rocksdb/utilities/options_util.h" #include "OpenRocksDb.h" diff --git a/extensions/rocksdb-repos/database/RocksDbInstance.h b/extensions/rocksdb-repos/database/RocksDbInstance.h index b59108d..63b031a 100644 --- a/extensions/rocksdb-repos/database/RocksDbInstance.h +++ b/extensions/rocksdb-repos/database/RocksDbInstance.h @@ -19,6 +19,8 @@ #pragma once #include <string> +#include <memory> +#include <unordered_map> #include "utils/OptionalUtils.h" #include "RocksDbUtils.h" #include "rocksdb/db.h" diff --git a/extensions/rocksdb-repos/database/RocksDbUtils.h b/extensions/rocksdb-repos/database/RocksDbUtils.h index 28e689c..ead059c 100644 --- a/extensions/rocksdb-repos/database/RocksDbUtils.h +++ b/extensions/rocksdb-repos/database/RocksDbUtils.h @@ -19,6 +19,7 @@ #pragma once #include <functional> +#include <algorithm> #include "rocksdb/db.h" #include "utils/GeneralUtils.h" diff --git a/extensions/rocksdb-repos/database/StringAppender.h b/extensions/rocksdb-repos/database/StringAppender.h index 8a242b7..7172a79 100644 --- a/extensions/rocksdb-repos/database/StringAppender.h +++ b/extensions/rocksdb-repos/database/StringAppender.h @@ -17,8 +17,11 @@ #pragma once -#include "rocksdb/merge_operator.h" #include <cstring> +#include <string> +#include <memory> +#include <algorithm> +#include "rocksdb/merge_operator.h" namespace org { namespace apache { @@ -29,9 +32,6 @@ namespace repository { class StringAppender : public rocksdb::AssociativeMergeOperator { public: - // Constructor: specify delimiter - explicit StringAppender() = default; - static std::shared_ptr<rocksdb::MergeOperator> transform(const std::shared_ptr<rocksdb::MergeOperator>& other) { if (other && std::strcmp(other->Name(), "StringAppender") == 0) { return other; diff --git a/extensions/script/CMakeLists.txt b/extensions/script/CMakeLists.txt index 34c26b6..800f0f8 100644 --- a/extensions/script/CMakeLists.txt +++ b/extensions/script/CMakeLists.txt @@ -77,4 +77,4 @@ endif() SET (SCRIPTING-EXTENSIONS minifi-script-extensions PARENT_SCOPE) register_extension(minifi-script-extensions) - +register_extension_linter(minifi-script-extensions-linter) diff --git a/extensions/script/ExecuteScript.h b/extensions/script/ExecuteScript.h index 476c631..9fab73f 100644 --- a/extensions/script/ExecuteScript.h +++ b/extensions/script/ExecuteScript.h @@ -18,12 +18,14 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_EXECUTESCRIPT_H -#define NIFI_MINIFI_CPP_EXECUTESCRIPT_H +#pragma once -#include <concurrentqueue.h> -#include <core/Resource.h> -#include <core/Processor.h> +#include <string> +#include <memory> + +#include "concurrentqueue.h" +#include "core/Resource.h" +#include "core/Processor.h" #include "ScriptEngine.h" #include "ScriptProcessContext.h" @@ -99,5 +101,3 @@ REGISTER_RESOURCE(ExecuteScript, "Executes a script given the flow file and a pr } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_EXECUTESCRIPT_H diff --git a/extensions/script/ScriptEngine.h b/extensions/script/ScriptEngine.h index 062e527..24de218 100644 --- a/extensions/script/ScriptEngine.h +++ b/extensions/script/ScriptEngine.h @@ -15,8 +15,7 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_SCRIPTENGINE_H -#define NIFI_MINIFI_CPP_SCRIPTENGINE_H +#pragma once #include <string> @@ -50,5 +49,3 @@ class ScriptEngine { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_SCRIPTENGINE_H diff --git a/extensions/script/ScriptException.h b/extensions/script/ScriptException.h index d3e36a0..f2b467f 100644 --- a/extensions/script/ScriptException.h +++ b/extensions/script/ScriptException.h @@ -17,14 +17,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SCRIPT_EXCEPTION_H__ -#define __SCRIPT_EXCEPTION_H__ +#pragma once -#include <sstream> #include <exception> +#include <utility> +#include <string> #include <stdexcept> -#include <errno.h> -#include <string.h> namespace org { namespace apache { @@ -40,8 +38,8 @@ class ScriptException : public std::exception { /*! * Create a new exception */ - ScriptException(std::string errorMsg) - : error_(std::move(errorMsg)) { + explicit ScriptException(std::string errorMsg) + : error_(std::move(errorMsg)) { } virtual ~ScriptException() noexcept = default; @@ -58,4 +56,3 @@ class ScriptException : public std::exception { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif diff --git a/extensions/script/ScriptFlowFile.cpp b/extensions/script/ScriptFlowFile.cpp index e38577c..a0f9155 100644 --- a/extensions/script/ScriptFlowFile.cpp +++ b/extensions/script/ScriptFlowFile.cpp @@ -20,7 +20,7 @@ #include <memory> #include <string> -#include <core/FlowFile.h> +#include "core/FlowFile.h" #include "ScriptFlowFile.h" diff --git a/extensions/script/ScriptFlowFile.h b/extensions/script/ScriptFlowFile.h index 1541065..2e054f6 100644 --- a/extensions/script/ScriptFlowFile.h +++ b/extensions/script/ScriptFlowFile.h @@ -16,10 +16,12 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H -#define NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H +#pragma once -#include <core/FlowFile.h> +#include <string> +#include <memory> + +#include "core/FlowFile.h" namespace org { namespace apache { @@ -47,5 +49,3 @@ class ScriptFlowFile { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H diff --git a/extensions/script/ScriptProcessContext.cpp b/extensions/script/ScriptProcessContext.cpp index cff4b15..f26009e 100644 --- a/extensions/script/ScriptProcessContext.cpp +++ b/extensions/script/ScriptProcessContext.cpp @@ -18,6 +18,7 @@ #include <memory> #include <string> +#include <utility> #include "ScriptProcessContext.h" diff --git a/extensions/script/ScriptProcessContext.h b/extensions/script/ScriptProcessContext.h index 7921ff1..1fbb8eb 100644 --- a/extensions/script/ScriptProcessContext.h +++ b/extensions/script/ScriptProcessContext.h @@ -16,12 +16,12 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H -#define NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H +#pragma once -#include <core/ProcessSession.h> +#include <string> +#include <memory> -#include "ScriptFlowFile.h" +#include "core/ProcessSession.h" namespace org { namespace apache { @@ -45,5 +45,3 @@ class ScriptProcessContext { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H diff --git a/extensions/script/lua/LuaBaseStream.h b/extensions/script/lua/LuaBaseStream.h index a57f219..b72fad4 100644 --- a/extensions/script/lua/LuaBaseStream.h +++ b/extensions/script/lua/LuaBaseStream.h @@ -15,12 +15,13 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_LUABASESTREAM_H -#define NIFI_MINIFI_CPP_LUABASESTREAM_H +#pragma once #include <memory> -#include <sol.hpp> -#include <io/BaseStream.h> +#include <string> + +#include "sol.hpp" +#include "io/BaseStream.h" namespace org { namespace apache { @@ -54,5 +55,3 @@ class LuaBaseStream { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_LUABASESTREAM_H diff --git a/extensions/script/lua/LuaProcessSession.cpp b/extensions/script/lua/LuaProcessSession.cpp index 3841dd4..4e20bb8 100644 --- a/extensions/script/lua/LuaProcessSession.cpp +++ b/extensions/script/lua/LuaProcessSession.cpp @@ -16,6 +16,7 @@ */ #include <memory> +#include <utility> #include "LuaProcessSession.h" diff --git a/extensions/script/lua/LuaProcessSession.h b/extensions/script/lua/LuaProcessSession.h index 67d558b..abe1a79 100644 --- a/extensions/script/lua/LuaProcessSession.h +++ b/extensions/script/lua/LuaProcessSession.h @@ -15,15 +15,15 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_LUAPROCESSSESSION_H -#define NIFI_MINIFI_CPP_LUAPROCESSSESSION_H +#pragma once -#include <sol.hpp> - -#include <core/ProcessSession.h> +#include <vector> +#include <memory> +#include "core/ProcessSession.h" #include "../ScriptFlowFile.h" +#include "sol.hpp" #include "LuaBaseStream.h" namespace org { @@ -95,5 +95,3 @@ class LuaProcessSession { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_LUAPROCESSSESSION_H diff --git a/extensions/script/lua/LuaScriptEngine.h b/extensions/script/lua/LuaScriptEngine.h index 13ebc5f..841c0d3 100644 --- a/extensions/script/lua/LuaScriptEngine.h +++ b/extensions/script/lua/LuaScriptEngine.h @@ -15,12 +15,15 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_LUASCRIPTENGINE_H -#define NIFI_MINIFI_CPP_LUASCRIPTENGINE_H +#pragma once #include <mutex> -#include <sol.hpp> -#include <core/ProcessSession.h> +#include <utility> +#include <string> +#include <memory> + +#include "sol.hpp" +#include "core/ProcessSession.h" #include "../ScriptEngine.h" #include "../ScriptProcessContext.h" @@ -105,5 +108,3 @@ class LuaScriptEngine : public script::ScriptEngine { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_LUASCRIPTENGINE_H diff --git a/extensions/script/python/PyBaseStream.h b/extensions/script/python/PyBaseStream.h index 1f11065..bc8130d 100644 --- a/extensions/script/python/PyBaseStream.h +++ b/extensions/script/python/PyBaseStream.h @@ -16,12 +16,12 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYBYTESTREAM_H -#define NIFI_MINIFI_CPP_PYBYTESTREAM_H +#pragma once -#include <pybind11/embed.h> +#include <memory> -#include <io/BaseStream.h> +#include "pybind11/embed.h" +#include "io/BaseStream.h" namespace org { namespace apache { @@ -48,5 +48,3 @@ class PyBaseStream { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_PYBYTESTREAM_H diff --git a/extensions/script/python/PyProcCreator.h b/extensions/script/python/PyProcCreator.h index d2cf5da..bf2c00e 100644 --- a/extensions/script/python/PyProcCreator.h +++ b/extensions/script/python/PyProcCreator.h @@ -15,10 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_PYPROCESS_PYPROCCREATOR_H_ -#define EXTENSIONS_PYPROCESS_PYPROCCREATOR_H_ +#pragma once #include <map> +#include <vector> +#include <string> +#include <memory> + #include "core/ClassLoader.h" #include "ExecutePythonProcessor.h" #include "utils/StringUtils.h" @@ -27,15 +30,11 @@ class PythonObjectFactory : public core::DefautObjectFactory<minifi::python::processors::ExecutePythonProcessor> { public: - explicit PythonObjectFactory(const std::string &file, const std::string &name) : file_(file), name_(name) { } - /** - * Create a shared pointer to a new processor. - */ std::shared_ptr<core::CoreComponent> create(const std::string &name) override { auto ptr = std::static_pointer_cast<minifi::python::processors::ExecutePythonProcessor>(DefautObjectFactory::create(name)); ptr->initialize(); @@ -43,9 +42,6 @@ class PythonObjectFactory : public core::DefautObjectFactory<minifi::python::pro return std::static_pointer_cast<core::CoreComponent>(ptr); } - /** - * Create a shared pointer to a new processor. - */ std::shared_ptr<core::CoreComponent> create(const std::string &name, const utils::Identifier &uuid) override { auto ptr = std::static_pointer_cast<minifi::python::processors::ExecutePythonProcessor>(DefautObjectFactory::create(name, uuid)); ptr->initialize(); @@ -53,9 +49,6 @@ class PythonObjectFactory : public core::DefautObjectFactory<minifi::python::pro return std::static_pointer_cast<core::CoreComponent>(ptr); } - /** - * Create a shared pointer to a new processor. - */ core::CoreComponent* createRaw(const std::string &name) override { auto ptr = dynamic_cast<minifi::python::processors::ExecutePythonProcessor*>(DefautObjectFactory::createRaw(name)); ptr->initialize(); @@ -63,15 +56,13 @@ class PythonObjectFactory : public core::DefautObjectFactory<minifi::python::pro return dynamic_cast<core::CoreComponent*>(ptr); } - /** - * Create a shared pointer to a new processor. - */ core::CoreComponent* createRaw(const std::string &name, const utils::Identifier &uuid) override { auto ptr = dynamic_cast<minifi::python::processors::ExecutePythonProcessor*>(DefautObjectFactory::createRaw(name, uuid)); ptr->initialize(); ptr->setProperty(minifi::python::processors::ExecutePythonProcessor::ScriptFile, file_); return dynamic_cast<core::CoreComponent*>(ptr); } + private: std::string file_; std::string name_; @@ -79,7 +70,6 @@ class PythonObjectFactory : public core::DefautObjectFactory<minifi::python::pro class PyProcCreator { public: - void addClassName(const std::string &name, std::string file) { file_mapping_[name] = file; } @@ -106,5 +96,3 @@ class PyProcCreator { }; #pragma GCC visibility pop - -#endif /* EXTENSIONS_PYPROCESS_PYPROCCREATOR_H_ */ diff --git a/extensions/script/python/PyProcessSession.cpp b/extensions/script/python/PyProcessSession.cpp index 2ccbc75..b0a5d36 100644 --- a/extensions/script/python/PyProcessSession.cpp +++ b/extensions/script/python/PyProcessSession.cpp @@ -16,9 +16,10 @@ * limitations under the License. */ +#include <utility> #include <memory> -#include <pybind11/embed.h> +#include "pybind11/embed.h" #include "PyProcessSession.h" diff --git a/extensions/script/python/PyProcessSession.h b/extensions/script/python/PyProcessSession.h index 33f61a3..1d6867e 100644 --- a/extensions/script/python/PyProcessSession.h +++ b/extensions/script/python/PyProcessSession.h @@ -16,15 +16,15 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYPROCESSESSION_H -#define NIFI_MINIFI_CPP_PYPROCESSESSION_H +#pragma once -#include <pybind11/embed.h> - -#include <core/ProcessSession.h> +#include <utility> +#include <memory> +#include <vector> +#include "pybind11/embed.h" +#include "core/ProcessSession.h" #include "../ScriptFlowFile.h" - #include "PyBaseStream.h" #pragma GCC visibility push(hidden) @@ -100,5 +100,3 @@ class PyProcessSession { } /* namespace org */ #pragma GCC visibility pop - -#endif // NIFI_MINIFI_CPP_PYPROCESSESSION_H diff --git a/extensions/script/python/PythonBindings.h b/extensions/script/python/PythonBindings.h index 11f0068..9f6bc27 100644 --- a/extensions/script/python/PythonBindings.h +++ b/extensions/script/python/PythonBindings.h @@ -19,13 +19,12 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYTHONBINDINGS_H -#define NIFI_MINIFI_CPP_PYTHONBINDINGS_H +#pragma once -#include <pybind11/embed.h> +#include <memory> -#include <core/ProcessSession.h> -#include <core/logging/LoggerConfiguration.h> +#include "pybind11/embed.h" +#include "core/ProcessSession.h" #include "../ScriptProcessContext.h" @@ -80,5 +79,3 @@ PYBIND11_EMBEDDED_MODULE(minifi_native, m) { // NOLINT .def("read", static_cast<py::bytes (python::PyBaseStream::*)(size_t)>(&python::PyBaseStream::read)) .def("write", &python::PyBaseStream::write); } - -#endif // NIFI_MINIFI_CPP_PYTHONBINDINGS_H diff --git a/extensions/script/python/PythonCreator.h b/extensions/script/python/PythonCreator.h index 0dde8ff..6d26e08 100644 --- a/extensions/script/python/PythonCreator.h +++ b/extensions/script/python/PythonCreator.h @@ -16,10 +16,11 @@ * limitations under the License. */ -#ifndef EXTENSIONS_SCRIPT_PYTHONCREATOR_H_ -#define EXTENSIONS_SCRIPT_PYTHONCREATOR_H_ +#pragma once #include <vector> +#include <utility> +#include <algorithm> #include <string> #include <memory> #include "core/Core.h" @@ -42,7 +43,6 @@ namespace python { */ class PythonCreator : public minifi::core::CoreComponent { public: - explicit PythonCreator(const std::string &name, const utils::Identifier &uuid = {}) : minifi::core::CoreComponent(name, uuid), logger_(logging::LoggerFactory<PythonCreator>::getLogger()) { @@ -176,13 +176,10 @@ class PythonCreator : public minifi::core::CoreComponent { std::vector<std::string> classpaths_; std::shared_ptr<logging::Logger> logger_; -} -; +}; } /* namespace python */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif /* EXTENSIONS_SCRIPT_PYTHONCREATOR_H_ */ diff --git a/extensions/script/python/PythonProcessor.cpp b/extensions/script/python/PythonProcessor.cpp index fd1fb98..c681231 100644 --- a/extensions/script/python/PythonProcessor.cpp +++ b/extensions/script/python/PythonProcessor.cpp @@ -17,8 +17,9 @@ */ #include <memory> +#include <string> -#include <pybind11/embed.h> +#include "pybind11/embed.h" #include "ExecutePythonProcessor.h" #include "PythonProcessor.h" diff --git a/extensions/script/python/PythonProcessor.h b/extensions/script/python/PythonProcessor.h index a86d703..bb6026a 100644 --- a/extensions/script/python/PythonProcessor.h +++ b/extensions/script/python/PythonProcessor.h @@ -16,16 +16,12 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYTHONPROCESSOR_H -#define NIFI_MINIFI_CPP_PYTHONPROCESSOR_H +#pragma once -#include <pybind11/embed.h> +#include <string> #include <memory> -#include <core/ProcessSession.h> -#include <core/Processor.h> - -#include "PyBaseStream.h" +#include "core/Processor.h" namespace org { namespace apache { @@ -58,7 +54,6 @@ class PythonProcessor { void releaseCoreResources(); private: - std::shared_ptr<core::Processor> processor_; }; @@ -67,5 +62,3 @@ class PythonProcessor { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // NIFI_MINIFI_CPP_PYTHONPROCESSOR_H diff --git a/extensions/script/python/PythonScriptEngine.h b/extensions/script/python/PythonScriptEngine.h index 65940e1..96a1195 100644 --- a/extensions/script/python/PythonScriptEngine.h +++ b/extensions/script/python/PythonScriptEngine.h @@ -15,13 +15,16 @@ * limitations under the License. */ -#ifndef NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H -#define NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H +#pragma once #include <mutex> -#include <pybind11/embed.h> -#include <core/ProcessSession.h> -#include <core/Processor.h> +#include <memory> +#include <string> +#include <utility> + +#include "pybind11/embed.h" +#include "core/ProcessSession.h" +#include "core/Processor.h" #include "../ScriptEngine.h" #include "../ScriptProcessContext.h" @@ -147,7 +150,7 @@ class PythonScriptEngine : public script::ScriptEngine { class TriggerSchedule { public: - TriggerSchedule(std::shared_ptr<script::ScriptProcessContext> script_context) + explicit TriggerSchedule(std::shared_ptr<script::ScriptProcessContext> script_context) : script_context_(script_context) { } @@ -235,5 +238,3 @@ class PythonScriptEngine : public script::ScriptEngine { } /* namespace org */ #pragma GCC visibility pop - -#endif // NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H diff --git a/extensions/script/pythonloader/PyProcLoader.h b/extensions/script/pythonloader/PyProcLoader.h index fa7b47c..57e6b7d 100644 --- a/extensions/script/pythonloader/PyProcLoader.h +++ b/extensions/script/pythonloader/PyProcLoader.h @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_PYPROC_H -#define EXTENSIONS_PYPROC_H +#pragma once #include <map> +#include <string> +#include <memory> +#include <vector> #include "core/ClassLoader.h" #include "ExecutePythonProcessor.h" #include "utils/StringUtils.h" @@ -59,10 +61,8 @@ class PyProcFactory : public core::ObjectFactory { } static bool added; -} -; +}; extern "C" { DLL_EXPORT void *createPyProcFactory(void); } -#endif /* EXTENSIONS_PYPROC_H */ diff --git a/extensions/sensors/CMakeLists.txt b/extensions/sensors/CMakeLists.txt index 18023f5..e330472 100644 --- a/extensions/sensors/CMakeLists.txt +++ b/extensions/sensors/CMakeLists.txt @@ -32,3 +32,4 @@ target_link_libraries(minifi-sensors RTIMULib::RTIMULib) SET (SENSOR-EXTENSIONS minifi-sensors PARENT_SCOPE) register_extension(minifi-sensors) +register_extension_linter(minifi-sensors-linter) diff --git a/extensions/sensors/GetEnvironmentalSensors.h b/extensions/sensors/GetEnvironmentalSensors.h index 6bbab46..d3557aa 100644 --- a/extensions/sensors/GetEnvironmentalSensors.h +++ b/extensions/sensors/GetEnvironmentalSensors.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_SENSORS_GETENVIRONMENTALSENSORS_H_ -#define EXTENSIONS_SENSORS_GETENVIRONMENTALSENSORS_H_ - - +#pragma once #include <memory> #include <regex> +#include <string> #include "utils/ByteArrayCallback.h" #include "FlowFileRecord.h" @@ -44,12 +42,11 @@ namespace processors { // EnvironmentalSensors Class class GetEnvironmentalSensors : public SensorBase { public: - // Constructor /*! * Create a new processor */ - GetEnvironmentalSensors(const std::string& name, const utils::Identifier& uuid = {}) + explicit GetEnvironmentalSensors(const std::string& name, const utils::Identifier& uuid = {}) : SensorBase(name, uuid), humidity_sensor_(nullptr), pressure_sensor_(nullptr), @@ -68,6 +65,7 @@ class GetEnvironmentalSensors : public SensorBase { protected: void notifyStop() override; + private: RTHumidity *humidity_sensor_; RTPressure *pressure_sensor_; @@ -82,4 +80,3 @@ REGISTER_RESOURCE(GetEnvironmentalSensors, "Provides sensor information from kno } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif /* EXTENSIONS_SENSORS_GETENVIRONMENTALSENSORS_H_ */ diff --git a/extensions/sensors/GetMovementSensors.h b/extensions/sensors/GetMovementSensors.h index ec0c84c..0ea1374 100644 --- a/extensions/sensors/GetMovementSensors.h +++ b/extensions/sensors/GetMovementSensors.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_SENSORS_GETMOVEMENT_H_ -#define EXTENSIONS_SENSORS_GETMOVEMENT_H_ - - +#pragma once #include <memory> #include <regex> +#include <string> #include "utils/ByteArrayCallback.h" #include "FlowFileRecord.h" @@ -44,12 +42,11 @@ namespace processors { // GetMovementSensors Class class GetMovementSensors : public SensorBase { public: - // Constructor /*! * Create a new processor */ - GetMovementSensors(const std::string& name, const utils::Identifier& uuid = {}) + explicit GetMovementSensors(const std::string& name, const utils::Identifier& uuid = {}) : SensorBase(name, uuid), logger_(logging::LoggerFactory<GetMovementSensors>::getLogger()) { } @@ -74,4 +71,3 @@ REGISTER_RESOURCE(GetMovementSensors, "Defines a processor that is able to retri } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif /* EXTENSIONS_SENSORS_GETMOVEMENT_H_ */ diff --git a/extensions/sensors/SensorBase.h b/extensions/sensors/SensorBase.h index 9b064b4..d633fab 100644 --- a/extensions/sensors/SensorBase.h +++ b/extensions/sensors/SensorBase.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_SENSORS_SENSORBASE_H_ -#define EXTENSIONS_SENSORS_SENSORBASE_H_ - - +#pragma once #include <memory> #include <regex> +#include <string> #include "utils/ByteArrayCallback.h" #include "FlowFileRecord.h" @@ -43,7 +41,6 @@ namespace processors { // SensorBase Class class SensorBase : public core::Processor { public: - // Constructor /*! * Create a new processor @@ -64,17 +61,18 @@ class SensorBase : public core::Processor { void onSchedule(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) override; class WriteCallback : public OutputStreamCallback { - public: - explicit WriteCallback(std::string data) - : data_{std::move(data)} - {} - std::string data_; - int64_t process(const std::shared_ptr<io::BaseStream>& stream) override { - if (data_.empty()) return 0; - const auto write_ret = stream->write(reinterpret_cast<const uint8_t*>(data_.data()), data_.size()); - return io::isError(write_ret) ? -1 : gsl::narrow<int64_t>(write_ret); - } - }; + public: + explicit WriteCallback(std::string data) + : data_{std::move(data)} { + } + std::string data_; + int64_t process(const std::shared_ptr<io::BaseStream>& stream) override { + if (data_.empty()) return 0; + const auto write_ret = stream->write(reinterpret_cast<const uint8_t*>(data_.data()), data_.size()); + return io::isError(write_ret) ? -1 : gsl::narrow<int64_t>(write_ret); + } + }; + protected: RTIMUSettings settings; std::unique_ptr<RTIMU> imu; @@ -86,4 +84,3 @@ class SensorBase : public core::Processor { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif /* EXTENSIONS_SENSORS_SENSORBASE_H_ */ diff --git a/extensions/sensors/SensorLoader.h b/extensions/sensors/SensorLoader.h index c0ac761..0da22fb 100644 --- a/extensions/sensors/SensorLoader.h +++ b/extensions/sensors/SensorLoader.h @@ -15,8 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_SENSORS_SENSORLOADER_H -#define EXTENSIONS_SENSORS_SENSORLOADER_H +#pragma once + +#include <vector> +#include <string> +#include <memory> #include "core/ClassLoader.h" #include "GetEnvironmentalSensors.h" @@ -59,10 +62,8 @@ class SensorFactory : public core::ObjectFactory { } static bool added; -} -; +}; extern "C" { DLL_EXPORT void *createSensorFactory(void); } -#endif // EXTENSIONS_SENSORS_SENSORLOADER_H diff --git a/extensions/sftp/CMakeLists.txt b/extensions/sftp/CMakeLists.txt index 6805730..7032fb2 100644 --- a/extensions/sftp/CMakeLists.txt +++ b/extensions/sftp/CMakeLists.txt @@ -30,3 +30,4 @@ target_link_libraries(minifi-sftp CURL::libcurl libssh2 RapidJSON) SET (SFTP minifi-sftp PARENT_SCOPE) register_extension(minifi-sftp) +register_extension_linter(minifi-sftp-linter) diff --git a/extensions/sftp/SFTPLoader.h b/extensions/sftp/SFTPLoader.h index ea27fac..4e29468 100644 --- a/extensions/sftp/SFTPLoader.h +++ b/extensions/sftp/SFTPLoader.h @@ -15,8 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSION_SFTPLOADER_H -#define EXTENSION_SFTPLOADER_H +#pragma once + +#include <vector> +#include <string> +#include <memory> #include "core/ClassLoader.h" #include "processors/PutSFTP.h" @@ -76,4 +79,3 @@ class SFTPFactory : public core::ObjectFactory { extern "C" { DLL_EXPORT void *createSFTPFactory(void); } -#endif /* EXTENSION_SFTPLOADER_H */ diff --git a/extensions/sftp/client/SFTPClient.cpp b/extensions/sftp/client/SFTPClient.cpp index cfa47ba..b94047a 100644 --- a/extensions/sftp/client/SFTPClient.cpp +++ b/extensions/sftp/client/SFTPClient.cpp @@ -22,9 +22,11 @@ #include <string> #include <exception> #include <sstream> -#include <iomanip> -#include "utils/StringUtils.h" +#include <algorithm> +#include <tuple> +#include <utility> +#include "utils/StringUtils.h" #include "utils/gsl.h" namespace org { @@ -35,7 +37,7 @@ namespace utils { #define SFTP_ERROR(CODE) case CODE: \ return #CODE -static const char* sftp_strerror(unsigned long err) { +static const char* sftp_strerror(unsigned long err) { // NOLINT(runtime/int) unsigned long comes from libssh2 API switch (err) { SFTP_ERROR(LIBSSH2_FX_OK); SFTP_ERROR(LIBSSH2_FX_EOF); @@ -64,7 +66,7 @@ static const char* sftp_strerror(unsigned long err) { } } -static SFTPError libssh2_sftp_error_to_sftp_error(unsigned long libssh2_sftp_error) { +static SFTPError libssh2_sftp_error_to_sftp_error(unsigned long libssh2_sftp_error) { // NOLINT(runtime/int) unsigned long comes from libssh2 API switch (libssh2_sftp_error) { case LIBSSH2_FX_OK: return SFTPError::Ok; @@ -106,7 +108,7 @@ LastSFTPError::LastSFTPError() , sftp_error_(SFTPError::Ok) { } -LastSFTPError& LastSFTPError::setLibssh2Error(unsigned long libssh2_sftp_error) { +LastSFTPError& LastSFTPError::setLibssh2Error(unsigned long libssh2_sftp_error) { // NOLINT(runtime/int) unsigned long comes from libssh2 API sftp_error_set_ = false; libssh2_sftp_error_ = libssh2_sftp_error; return *this; @@ -118,7 +120,7 @@ LastSFTPError& LastSFTPError::setSftpError(const SFTPError& sftp_error) { return *this; } -LastSFTPError::operator unsigned long() const { +LastSFTPError::operator unsigned long() const { // NOLINT(runtime/int) unsigned long comes from libssh2 API if (sftp_error_set_) { return LIBSSH2_FX_OK; } else { @@ -289,7 +291,7 @@ bool SFTPClient::connect() { /* Only CURLINFO_ACTIVESOCKET works on Win64 */ curl_res = curl_easy_getinfo(easy_, CURLINFO_ACTIVESOCKET, &sockfd); #else - long sockfd; + long sockfd; // NOLINT(runtime/int) long due to libcurl API /* Some older cURL versions only support CURLINFO_LASTSOCKET */ curl_res = curl_easy_getinfo(easy_, CURLINFO_LASTSOCKET, &sockfd); #endif diff --git a/extensions/sftp/client/SFTPClient.h b/extensions/sftp/client/SFTPClient.h index 6265573..c07bdd4 100644 --- a/extensions/sftp/client/SFTPClient.h +++ b/extensions/sftp/client/SFTPClient.h @@ -15,8 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SFTP_CLIENT_H__ -#define __SFTP_CLIENT_H__ +#pragma once #include <curl/curl.h> #include <libssh2.h> @@ -24,15 +23,13 @@ #include <vector> #include <iostream> #include <string> -#include <vector> +#include <tuple> +#include <memory> #include "Exception.h" #include "utils/Enum.h" #include "utils/HTTPClient.h" #include "core/logging/Logger.h" -#include "core/logging/LoggerConfiguration.h" -#include "properties/Configure.h" -#include "io/validation.h" #include "io/BaseStream.h" namespace org { @@ -70,20 +67,19 @@ class LastSFTPError { LastSFTPError& operator=(const LastSFTPError&) = delete; LastSFTPError& operator=(LastSFTPError&&) = delete; - LastSFTPError& setLibssh2Error(unsigned long libssh2_sftp_error); + LastSFTPError& setLibssh2Error(unsigned long libssh2_sftp_error); // NOLINT(runtime/int) unsigned long comes from libssh2 API LastSFTPError& setSftpError(const SFTPError& sftp_error); - operator unsigned long() const; + operator unsigned long() const; // NOLINT(runtime/int) unsigned long comes from libssh2 API operator SFTPError() const; private: bool sftp_error_set_; - unsigned long libssh2_sftp_error_; + unsigned long libssh2_sftp_error_; // NOLINT(runtime/int) unsigned long comes from libssh2 API SFTPError sftp_error_; }; class SFTPClient { public: - SFTPClient(const std::string &hostname, uint16_t port, const std::string& username); ~SFTPClient(); @@ -203,5 +199,3 @@ class SFTPClient { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/sftp/processors/FetchSFTP.cpp b/extensions/sftp/processors/FetchSFTP.cpp index e50e1f1..d7fcf9b 100644 --- a/extensions/sftp/processors/FetchSFTP.cpp +++ b/extensions/sftp/processors/FetchSFTP.cpp @@ -46,41 +46,50 @@ namespace nifi { namespace minifi { namespace processors { -core::Property FetchSFTP::RemoteFile( - core::PropertyBuilder::createProperty("Remote File")->withDescription("The fully qualified filename on the remote system") - ->isRequired(true)->supportsExpressionLanguage(true)->build()); +core::Property FetchSFTP::RemoteFile(core::PropertyBuilder::createProperty("Remote File") + ->withDescription("The fully qualified filename on the remote system") + ->isRequired(true)->supportsExpressionLanguage(true)->build()); + core::Property FetchSFTP::CompletionStrategy( - core::PropertyBuilder::createProperty("Completion Strategy")->withDescription("Specifies what to do with the original file on the server once it has been pulled into NiFi. If the Completion Strategy fails, a warning will be logged but the data will still be transferred.") - ->isRequired(true) - ->withAllowableValues<std::string>({COMPLETION_STRATEGY_NONE, - COMPLETION_STRATEGY_MOVE_FILE, - COMPLETION_STRATEGY_DELETE_FILE}) - ->withDefaultValue(COMPLETION_STRATEGY_NONE)->build()); -core::Property FetchSFTP::MoveDestinationDirectory( - core::PropertyBuilder::createProperty("Move Destination Directory")->withDescription("The directory on the remote server to move the original file to once it has been ingested into NiFi. " - "This property is ignored unless the Completion Strategy is set to 'Move File'. " - "The specified directory must already exist on the remote system if 'Create Directory' is disabled, or the rename will fail.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property FetchSFTP::CreateDirectory( - core::PropertyBuilder::createProperty("Create Directory")->withDescription("Specifies whether or not the remote directory should be created if it does not exist.") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); -core::Property FetchSFTP::DisableDirectoryListing( - core::PropertyBuilder::createProperty("Disable Directory Listing")->withDescription("Control how 'Move Destination Directory' is created when 'Completion Strategy' is 'Move File' and 'Create Directory' is enabled. " - "If set to 'true', directory listing is not performed prior to create missing directories. " - "By default, this processor executes a directory listing command to see target directory existence before creating missing directories. " - "However, there are situations that you might need to disable the directory listing such as the following. " - "Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. " - "Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, " - "then an error is returned because the directory already exists.") - ->isRequired(false)->withDefaultValue<bool>(false)->build()); -core::Property FetchSFTP::UseCompression( - core::PropertyBuilder::createProperty("Use Compression")->withDescription("Indicates whether or not ZLIB compression should be used when transferring files") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); - -core::Relationship FetchSFTP::Success("success", "All FlowFiles that are received are routed to success"); -core::Relationship FetchSFTP::CommsFailure("comms.failure", "Any FlowFile that could not be fetched from the remote server due to a communications failure will be transferred to this Relationship."); -core::Relationship FetchSFTP::NotFound("not.found", "Any FlowFile for which we receive a 'Not Found' message from the remote server will be transferred to this Relationship."); -core::Relationship FetchSFTP::PermissionDenied("permission.denied", "Any FlowFile that could not be fetched from the remote server due to insufficient permissions will be transferred to this Relationship."); + core::PropertyBuilder::createProperty("Completion Strategy") + ->withDescription("Specifies what to do with the original file on the server once it has been pulled into NiFi. " + "If the Completion Strategy fails, a warning will be logged but the data will still be transferred.") + ->isRequired(true) + ->withAllowableValues<std::string>({COMPLETION_STRATEGY_NONE, COMPLETION_STRATEGY_MOVE_FILE, COMPLETION_STRATEGY_DELETE_FILE}) + ->withDefaultValue(COMPLETION_STRATEGY_NONE)->build()); + +core::Property FetchSFTP::MoveDestinationDirectory(core::PropertyBuilder::createProperty("Move Destination Directory") + ->withDescription("The directory on the remote server to move the original file to once it has been ingested into NiFi. " + "This property is ignored unless the Completion Strategy is set to 'Move File'. " + "The specified directory must already exist on the remote system if 'Create Directory' is disabled, or the rename will fail.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property FetchSFTP::CreateDirectory(core::PropertyBuilder::createProperty("Create Directory") + ->withDescription("Specifies whether or not the remote directory should be created if it does not exist.") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Property FetchSFTP::DisableDirectoryListing(core::PropertyBuilder::createProperty("Disable Directory Listing") + ->withDescription("Control how 'Move Destination Directory' is created when 'Completion Strategy' is 'Move File' and 'Create Directory' is enabled. " + "If set to 'true', directory listing is not performed prior to create missing directories. " + "By default, this processor executes a directory listing command to see target directory existence before creating missing directories. " + "However, there are situations that you might need to disable the directory listing such as the following. " + "Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. " + "Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, " + "then an error is returned because the directory already exists.") + ->isRequired(false)->withDefaultValue<bool>(false)->build()); + +core::Property FetchSFTP::UseCompression(core::PropertyBuilder::createProperty("Use Compression") + ->withDescription("Indicates whether or not ZLIB compression should be used when transferring files") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Relationship FetchSFTP::Success("success", + "All FlowFiles that are received are routed to success"); +core::Relationship FetchSFTP::CommsFailure("comms.failure", + "Any FlowFile that could not be fetched from the remote server due to a communications failure will be transferred to this Relationship."); +core::Relationship FetchSFTP::NotFound("not.found", + "Any FlowFile for which we receive a 'Not Found' message from the remote server will be transferred to this Relationship."); +core::Relationship FetchSFTP::PermissionDenied("permission.denied", + "Any FlowFile that could not be fetched from the remote server due to insufficient permissions will be transferred to this Relationship."); constexpr char const* FetchSFTP::COMPLETION_STRATEGY_NONE; constexpr char const* FetchSFTP::COMPLETION_STRATEGY_MOVE_FILE; diff --git a/extensions/sftp/processors/FetchSFTP.h b/extensions/sftp/processors/FetchSFTP.h index 381b268..2da6fb0 100644 --- a/extensions/sftp/processors/FetchSFTP.h +++ b/extensions/sftp/processors/FetchSFTP.h @@ -14,8 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __FETCH_SFTP_H__ -#define __FETCH_SFTP_H__ +#pragma once #include <memory> #include <string> @@ -40,7 +39,6 @@ namespace processors { class FetchSFTP : public SFTPProcessorBase { public: - static constexpr char const *COMPLETION_STRATEGY_NONE = "None"; static constexpr char const *COMPLETION_STRATEGY_MOVE_FILE = "Move File"; static constexpr char const *COMPLETION_STRATEGY_DELETE_FILE = "Delete File"; @@ -51,7 +49,7 @@ class FetchSFTP : public SFTPProcessorBase { /*! * Create a new processor */ - FetchSFTP(const std::string& name, const utils::Identifier& uuid = {}); + explicit FetchSFTP(const std::string& name, const utils::Identifier& uuid = {}); virtual ~FetchSFTP(); // Supported Properties @@ -107,5 +105,3 @@ REGISTER_RESOURCE(FetchSFTP, "Fetches the content of a file from a remote SFTP s } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/sftp/processors/ListSFTP.cpp b/extensions/sftp/processors/ListSFTP.cpp index 858d8e3..e6c008b 100644 --- a/extensions/sftp/processors/ListSFTP.cpp +++ b/extensions/sftp/processors/ListSFTP.cpp @@ -59,73 +59,85 @@ namespace nifi { namespace minifi { namespace processors { -core::Property ListSFTP::ListingStrategy( - core::PropertyBuilder::createProperty("Listing Strategy")->withDescription("Specify how to determine new/updated entities. See each strategy descriptions for detail.") - ->isRequired(true) - ->withAllowableValues<std::string>({LISTING_STRATEGY_TRACKING_TIMESTAMPS, - LISTING_STRATEGY_TRACKING_ENTITIES}) - ->withDefaultValue(LISTING_STRATEGY_TRACKING_TIMESTAMPS)->build()); -core::Property ListSFTP::RemotePath( - core::PropertyBuilder::createProperty("Remote Path")->withDescription("The fully qualified filename on the remote system") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property ListSFTP::SearchRecursively( - core::PropertyBuilder::createProperty("Search Recursively")->withDescription("If true, will pull files from arbitrarily nested subdirectories; " - "otherwise, will not traverse subdirectories") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); -core::Property ListSFTP::FollowSymlink( - core::PropertyBuilder::createProperty("Follow symlink")->withDescription("If true, will pull even symbolic files and also nested symbolic subdirectories; " - "otherwise, will not read symbolic files and will not traverse symbolic link subdirectories") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); -core::Property ListSFTP::FileFilterRegex( - core::PropertyBuilder::createProperty("File Filter Regex")->withDescription("Provides a Java Regular Expression for filtering Filenames; " - "if a filter is supplied, only files whose names match that Regular Expression will be fetched") - ->isRequired(false)->build()); -core::Property ListSFTP::PathFilterRegex( - core::PropertyBuilder::createProperty("Path Filter Regex")->withDescription("When Search Recursively is true, then only subdirectories whose path matches the given Regular Expression will be scanned") - ->isRequired(false)->build()); -core::Property ListSFTP::IgnoreDottedFiles( - core::PropertyBuilder::createProperty("Ignore Dotted Files")->withDescription("If true, files whose names begin with a dot (\".\") will be ignored") - ->isRequired(true)->withDefaultValue<bool>(true)->build()); -core::Property ListSFTP::TargetSystemTimestampPrecision( - core::PropertyBuilder::createProperty("Target System Timestamp Precision")->withDescription("Specify timestamp precision at the target system. " - "Since this processor uses timestamp of entities to decide which should be listed, " - "it is crucial to use the right timestamp precision.") - ->isRequired(true) - ->withAllowableValues<std::string>({TARGET_SYSTEM_TIMESTAMP_PRECISION_AUTO_DETECT, +core::Property ListSFTP::ListingStrategy(core::PropertyBuilder::createProperty("Listing Strategy") + ->withDescription("Specify how to determine new/updated entities. See each strategy descriptions for detail.") + ->isRequired(true) + ->withAllowableValues<std::string>({LISTING_STRATEGY_TRACKING_TIMESTAMPS, LISTING_STRATEGY_TRACKING_ENTITIES}) + ->withDefaultValue(LISTING_STRATEGY_TRACKING_TIMESTAMPS)->build()); + +core::Property ListSFTP::RemotePath(core::PropertyBuilder::createProperty("Remote Path") + ->withDescription("The fully qualified filename on the remote system") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property ListSFTP::SearchRecursively(core::PropertyBuilder::createProperty("Search Recursively") + ->withDescription("If true, will pull files from arbitrarily nested subdirectories; " + "otherwise, will not traverse subdirectories") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Property ListSFTP::FollowSymlink(core::PropertyBuilder::createProperty("Follow symlink") + ->withDescription("If true, will pull even symbolic files and also nested symbolic subdirectories; " + "otherwise, will not read symbolic files and will not traverse symbolic link subdirectories") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Property ListSFTP::FileFilterRegex(core::PropertyBuilder::createProperty("File Filter Regex") + ->withDescription("Provides a Java Regular Expression for filtering Filenames; " + "if a filter is supplied, only files whose names match that Regular Expression will be fetched") + ->isRequired(false)->build()); + +core::Property ListSFTP::PathFilterRegex(core::PropertyBuilder::createProperty("Path Filter Regex") + ->withDescription("When Search Recursively is true, then only subdirectories whose path matches the given Regular Expression will be scanned") + ->isRequired(false)->build()); + +core::Property ListSFTP::IgnoreDottedFiles(core::PropertyBuilder::createProperty("Ignore Dotted Files") + ->withDescription("If true, files whose names begin with a dot (\".\") will be ignored") + ->isRequired(true)->withDefaultValue<bool>(true)->build()); + +core::Property ListSFTP::TargetSystemTimestampPrecision(core::PropertyBuilder::createProperty("Target System Timestamp Precision") + ->withDescription("Specify timestamp precision at the target system. " + "Since this processor uses timestamp of entities to decide which should be listed, " + "it is crucial to use the right timestamp precision.") + ->isRequired(true) + ->withAllowableValues<std::string>({TARGET_SYSTEM_TIMESTAMP_PRECISION_AUTO_DETECT, TARGET_SYSTEM_TIMESTAMP_PRECISION_MILLISECONDS, TARGET_SYSTEM_TIMESTAMP_PRECISION_SECONDS, TARGET_SYSTEM_TIMESTAMP_PRECISION_MINUTES}) - ->withDefaultValue(TARGET_SYSTEM_TIMESTAMP_PRECISION_AUTO_DETECT)->build()); -core::Property ListSFTP::EntityTrackingTimeWindow( - core::PropertyBuilder::createProperty("Entity Tracking Time Window")->withDescription("Specify how long this processor should track already-listed entities. " - "'Tracking Entities' strategy can pick any entity whose timestamp is inside the specified time window. " - "For example, if set to '30 minutes', any entity having timestamp in recent 30 minutes will be the listing target when this processor runs. " - "A listed entity is considered 'new/updated' and a FlowFile is emitted if one of following condition meets: " - "1. does not exist in the already-listed entities, " - "2. has newer timestamp than the cached entity, " - "3. has different size than the cached entity. " - "If a cached entity's timestamp becomes older than specified time window, that entity will be removed from the cached already-listed entities. " - "Used by 'Tracking Entities' strategy.") - ->isRequired(false)->build()); -core::Property ListSFTP::EntityTrackingInitialListingTarget( - core::PropertyBuilder::createProperty("Entity Tracking Initial Listing Target")->withDescription("Specify how initial listing should be handled. Used by 'Tracking Entities' strategy.") - ->withAllowableValues<std::string>({ENTITY_TRACKING_INITIAL_LISTING_TARGET_TRACKING_TIME_WINDOW, + ->withDefaultValue(TARGET_SYSTEM_TIMESTAMP_PRECISION_AUTO_DETECT)->build()); + +core::Property ListSFTP::EntityTrackingTimeWindow(core::PropertyBuilder::createProperty("Entity Tracking Time Window") + ->withDescription("Specify how long this processor should track already-listed entities. " + "'Tracking Entities' strategy can pick any entity whose timestamp is inside the specified time window. " + "For example, if set to '30 minutes', any entity having timestamp in recent 30 minutes will be the listing target when this processor runs. " + "A listed entity is considered 'new/updated' and a FlowFile is emitted if one of following condition meets: " + "1. does not exist in the already-listed entities, " + "2. has newer timestamp than the cached entity, " + "3. has different size than the cached entity. " + "If a cached entity's timestamp becomes older than specified time window, that entity will be removed from the cached already-listed entities. " + "Used by 'Tracking Entities' strategy.") + ->isRequired(false)->build()); + +core::Property ListSFTP::EntityTrackingInitialListingTarget(core::PropertyBuilder::createProperty("Entity Tracking Initial Listing Target") + ->withDescription("Specify how initial listing should be handled. Used by 'Tracking Entities' strategy.") + ->withAllowableValues<std::string>({ENTITY_TRACKING_INITIAL_LISTING_TARGET_TRACKING_TIME_WINDOW, ENTITY_TRACKING_INITIAL_LISTING_TARGET_ALL_AVAILABLE}) - ->isRequired(false)->withDefaultValue(ENTITY_TRACKING_INITIAL_LISTING_TARGET_ALL_AVAILABLE)->build()); -core::Property ListSFTP::MinimumFileAge( - core::PropertyBuilder::createProperty("Minimum File Age")->withDescription("The minimum age that a file must be in order to be pulled; " - "any file younger than this amount of time (according to last modification date) will be ignored") - ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("0 sec")->build()); -core::Property ListSFTP::MaximumFileAge( - core::PropertyBuilder::createProperty("Maximum File Age")->withDescription("The maximum age that a file must be in order to be pulled; " - "any file older than this amount of time (according to last modification date) will be ignored") - ->isRequired(false)->build()); -core::Property ListSFTP::MinimumFileSize( - core::PropertyBuilder::createProperty("Minimum File Size")->withDescription("The minimum size that a file must be in order to be pulled") - ->isRequired(true)->withDefaultValue<core::DataSizeValue>("0 B")->build()); -core::Property ListSFTP::MaximumFileSize( - core::PropertyBuilder::createProperty("Maximum File Size")->withDescription("The maximum size that a file must be in order to be pulled") - ->isRequired(false)->build()); + ->isRequired(false)->withDefaultValue(ENTITY_TRACKING_INITIAL_LISTING_TARGET_ALL_AVAILABLE)->build()); + +core::Property ListSFTP::MinimumFileAge(core::PropertyBuilder::createProperty("Minimum File Age") + ->withDescription("The minimum age that a file must be in order to be pulled; " + "any file younger than this amount of time (according to last modification date) will be ignored") + ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("0 sec")->build()); + +core::Property ListSFTP::MaximumFileAge(core::PropertyBuilder::createProperty("Maximum File Age") + ->withDescription("The maximum age that a file must be in order to be pulled; " + "any file older than this amount of time (according to last modification date) will be ignored") + ->isRequired(false)->build()); + +core::Property ListSFTP::MinimumFileSize(core::PropertyBuilder::createProperty("Minimum File Size") + ->withDescription("The minimum size that a file must be in order to be pulled") + ->isRequired(true)->withDefaultValue<core::DataSizeValue>("0 B")->build()); + +core::Property ListSFTP::MaximumFileSize(core::PropertyBuilder::createProperty("Maximum File Size") + ->withDescription("The maximum size that a file must be in order to be pulled") + ->isRequired(false)->build()); core::Relationship ListSFTP::Success("success", "All FlowFiles that are received are routed to success"); diff --git a/extensions/sftp/processors/ListSFTP.h b/extensions/sftp/processors/ListSFTP.h index 5ec9eef..5a32b58 100644 --- a/extensions/sftp/processors/ListSFTP.h +++ b/extensions/sftp/processors/ListSFTP.h @@ -14,8 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __LIST_SFTP_H__ -#define __LIST_SFTP_H__ +#pragma once #include <memory> #include <string> @@ -23,20 +22,18 @@ #include <chrono> #include <cstdint> #include <unordered_map> +#include <set> +#include <tuple> +#include <vector> #include "SFTPProcessorBase.h" -#include "utils/ByteArrayCallback.h" -#include "FlowFileRecord.h" #include "core/Processor.h" #include "core/ProcessSession.h" -#include "core/Core.h" #include "core/Property.h" #include "core/Resource.h" -#include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" #include "utils/RegexUtils.h" #include "controllers/keyvalue/PersistableKeyValueStoreService.h" -#include "../client/SFTPClient.h" namespace org { namespace apache { @@ -46,7 +43,6 @@ namespace processors { class ListSFTP : public SFTPProcessorBase { public: - static constexpr char const *LISTING_STRATEGY_TRACKING_TIMESTAMPS = "Tracking Timestamps"; static constexpr char const *LISTING_STRATEGY_TRACKING_ENTITIES = "Tracking Entities"; @@ -64,7 +60,7 @@ class ListSFTP : public SFTPProcessorBase { /*! * Create a new processor */ - ListSFTP(const std::string& name, const utils::Identifier& uuid = {}); + explicit ListSFTP(const std::string& name, const utils::Identifier& uuid = {}); virtual ~ListSFTP(); // Supported Properties @@ -208,5 +204,3 @@ REGISTER_RESOURCE(ListSFTP, "Performs a listing of the files residing on an SFTP } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/sftp/processors/PutSFTP.cpp b/extensions/sftp/processors/PutSFTP.cpp index 47436cb..16d0b84 100644 --- a/extensions/sftp/processors/PutSFTP.cpp +++ b/extensions/sftp/processors/PutSFTP.cpp @@ -49,71 +49,84 @@ namespace nifi { namespace minifi { namespace processors { -core::Property PutSFTP::RemotePath( - core::PropertyBuilder::createProperty("Remote Path")->withDescription("The path on the remote system from which to pull or push files") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::CreateDirectory( - core::PropertyBuilder::createProperty("Create Directory")->withDescription("Specifies whether or not the remote directory should be created if it does not exist.") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); -core::Property PutSFTP::DisableDirectoryListing( - core::PropertyBuilder::createProperty("Disable Directory Listing")->withDescription("If set to 'true', directory listing is not performed prior to create missing directories. " - "By default, this processor executes a directory listing command to see target directory existence before creating missing directories. " - "However, there are situations that you might need to disable the directory listing such as the following. " - "Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. " - "Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, " - "then an error is returned because the directory already exists.") - ->isRequired(false)->withDefaultValue<bool>(false)->build()); -core::Property PutSFTP::BatchSize( - core::PropertyBuilder::createProperty("Batch Size")->withDescription("The maximum number of FlowFiles to send in a single connection") - ->isRequired(true)->withDefaultValue<uint64_t>(500)->build()); -core::Property PutSFTP::ConflictResolution( - core::PropertyBuilder::createProperty("Conflict Resolution")->withDescription("Determines how to handle the problem of filename collisions") - ->isRequired(true) - ->withAllowableValues<std::string>({CONFLICT_RESOLUTION_REPLACE, - CONFLICT_RESOLUTION_IGNORE, - CONFLICT_RESOLUTION_RENAME, - CONFLICT_RESOLUTION_REJECT, - CONFLICT_RESOLUTION_FAIL, - CONFLICT_RESOLUTION_NONE}) - ->withDefaultValue(CONFLICT_RESOLUTION_NONE)->build()); -core::Property PutSFTP::RejectZeroByte( - core::PropertyBuilder::createProperty("Reject Zero-Byte Files")->withDescription("Determines whether or not Zero-byte files should be rejected without attempting to transfer") - ->isRequired(false)->withDefaultValue<bool>(true)->build()); -core::Property PutSFTP::DotRename( - core::PropertyBuilder::createProperty("Dot Rename")->withDescription("If true, then the filename of the sent file is prepended with a \".\" and then renamed back to the original once the file is completely sent. " - "Otherwise, there is no rename. This property is ignored if the Temporary Filename property is set.") - ->isRequired(false)->withDefaultValue<bool>(true)->build()); -core::Property PutSFTP::TempFilename( - core::PropertyBuilder::createProperty("Temporary Filename")->withDescription("If set, the filename of the sent file will be equal to the value specified during the transfer and after successful completion will be renamed to the original filename. " - "If this value is set, the Dot Rename property is ignored.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::LastModifiedTime( - core::PropertyBuilder::createProperty("Last Modified Time")->withDescription("The lastModifiedTime to assign to the file after transferring it. " - "If not set, the lastModifiedTime will not be changed. " - "Format must be yyyy-MM-dd'T'HH:mm:ssZ. " - "You may also use expression language such as ${file.lastModifiedTime}. " - "If the value is invalid, the processor will not be invalid but will fail to change lastModifiedTime of the file.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::Permissions( - core::PropertyBuilder::createProperty("Permissions")->withDescription("The permissions to assign to the file after transferring it. " - "Format must be either UNIX rwxrwxrwx with a - in place of denied permissions (e.g. rw-r--r--) or an octal number (e.g. 644). " - "If not set, the permissions will not be changed. " - "You may also use expression language such as ${file.permissions}. " - "If the value is invalid, the processor will not be invalid but will fail to change permissions of the file.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::RemoteOwner( - core::PropertyBuilder::createProperty("Remote Owner")->withDescription("Integer value representing the User ID to set on the file after transferring it. " - "If not set, the owner will not be set. You may also use expression language such as ${file.owner}. " - "If the value is invalid, the processor will not be invalid but will fail to change the owner of the file.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::RemoteGroup( - core::PropertyBuilder::createProperty("Remote Group")->withDescription("Integer value representing the Group ID to set on the file after transferring it. " - "If not set, the group will not be set. You may also use expression language such as ${file.group}. " - "If the value is invalid, the processor will not be invalid but will fail to change the group of the file.") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property PutSFTP::UseCompression( - core::PropertyBuilder::createProperty("Use Compression")->withDescription("Indicates whether or not ZLIB compression should be used when transferring files") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); +core::Property PutSFTP::RemotePath(core::PropertyBuilder::createProperty("Remote Path") + ->withDescription("The path on the remote system from which to pull or push files") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::CreateDirectory(core::PropertyBuilder::createProperty("Create Directory") + ->withDescription("Specifies whether or not the remote directory should be created if it does not exist.") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Property PutSFTP::DisableDirectoryListing(core::PropertyBuilder::createProperty("Disable Directory Listing") + ->withDescription("If set to 'true', directory listing is not performed prior to create missing directories. " + "By default, this processor executes a directory listing command to see target directory existence before creating missing directories. " + "However, there are situations that you might need to disable the directory listing such as the following. " + "Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. " + "Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, " + "then an error is returned because the directory already exists.") + ->isRequired(false)->withDefaultValue<bool>(false)->build()); + +core::Property PutSFTP::BatchSize(core::PropertyBuilder::createProperty("Batch Size") + ->withDescription("The maximum number of FlowFiles to send in a single connection") + ->isRequired(true)->withDefaultValue<uint64_t>(500)->build()); + +core::Property PutSFTP::ConflictResolution(core::PropertyBuilder::createProperty("Conflict Resolution") + ->withDescription("Determines how to handle the problem of filename collisions") + ->isRequired(true) + ->withAllowableValues<std::string>({CONFLICT_RESOLUTION_REPLACE, + CONFLICT_RESOLUTION_IGNORE, + CONFLICT_RESOLUTION_RENAME, + CONFLICT_RESOLUTION_REJECT, + CONFLICT_RESOLUTION_FAIL, + CONFLICT_RESOLUTION_NONE}) + ->withDefaultValue(CONFLICT_RESOLUTION_NONE)->build()); + +core::Property PutSFTP::RejectZeroByte(core::PropertyBuilder::createProperty("Reject Zero-Byte Files") + ->withDescription("Determines whether or not Zero-byte files should be rejected without attempting to transfer") + ->isRequired(false)->withDefaultValue<bool>(true)->build()); + +core::Property PutSFTP::DotRename(core::PropertyBuilder::createProperty("Dot Rename") + ->withDescription("If true, then the filename of the sent file is prepended with a \".\" and then renamed back to the original once the file is completely sent. " + "Otherwise, there is no rename. This property is ignored if the Temporary Filename property is set.") + ->isRequired(false)->withDefaultValue<bool>(true)->build()); + +core::Property PutSFTP::TempFilename(core::PropertyBuilder::createProperty("Temporary Filename") + ->withDescription("If set, the filename of the sent file will be equal to the value specified during the transfer and after successful completion will be renamed to the original filename. " + "If this value is set, the Dot Rename property is ignored.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::LastModifiedTime(core::PropertyBuilder::createProperty("Last Modified Time") + ->withDescription("The lastModifiedTime to assign to the file after transferring it. " + "If not set, the lastModifiedTime will not be changed. " + "Format must be yyyy-MM-dd'T'HH:mm:ssZ. " + "You may also use expression language such as ${file.lastModifiedTime}. " + "If the value is invalid, the processor will not be invalid but will fail to change lastModifiedTime of the file.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::Permissions(core::PropertyBuilder::createProperty("Permissions") + ->withDescription("The permissions to assign to the file after transferring it. " + "Format must be either UNIX rwxrwxrwx with a - in place of denied permissions (e.g. rw-r--r--) or an octal number (e.g. 644). " + "If not set, the permissions will not be changed. " + "You may also use expression language such as ${file.permissions}. " + "If the value is invalid, the processor will not be invalid but will fail to change permissions of the file.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::RemoteOwner(core::PropertyBuilder::createProperty("Remote Owner") + ->withDescription("Integer value representing the User ID to set on the file after transferring it. " + "If not set, the owner will not be set. You may also use expression language such as ${file.owner}. " + "If the value is invalid, the processor will not be invalid but will fail to change the owner of the file.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::RemoteGroup(core::PropertyBuilder::createProperty("Remote Group") + ->withDescription("Integer value representing the Group ID to set on the file after transferring it. " + "If not set, the group will not be set. You may also use expression language such as ${file.group}. " + "If the value is invalid, the processor will not be invalid but will fail to change the group of the file.") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property PutSFTP::UseCompression(core::PropertyBuilder::createProperty("Use Compression") + ->withDescription("Indicates whether or not ZLIB compression should be used when transferring files") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + core::Relationship PutSFTP::Success("success", "FlowFiles that are successfully sent will be routed to success"); core::Relationship PutSFTP::Reject("reject", "FlowFiles that were rejected by the destination system"); diff --git a/extensions/sftp/processors/PutSFTP.h b/extensions/sftp/processors/PutSFTP.h index 6f5703a..76c13ee 100644 --- a/extensions/sftp/processors/PutSFTP.h +++ b/extensions/sftp/processors/PutSFTP.h @@ -16,8 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __PUT_SFTP_H__ -#define __PUT_SFTP_H__ +#pragma once #include <memory> #include <string> @@ -45,9 +44,8 @@ namespace nifi { namespace minifi { namespace processors { - class PutSFTP : public SFTPProcessorBase { +class PutSFTP : public SFTPProcessorBase { public: - static constexpr char const *CONFLICT_RESOLUTION_REPLACE = "REPLACE"; static constexpr char const *CONFLICT_RESOLUTION_IGNORE = "IGNORE"; static constexpr char const *CONFLICT_RESOLUTION_RENAME = "RENAME"; @@ -61,7 +59,7 @@ namespace processors { /*! * Create a new processor */ - PutSFTP(const std::string& name, const utils::Identifier& uuid = {}); + explicit PutSFTP(const std::string& name, const utils::Identifier& uuid = {}); virtual ~PutSFTP(); // Supported Properties @@ -108,9 +106,9 @@ namespace processors { }; private: - core::annotation::Input getInputRequirement() const override { - return core::annotation::Input::INPUT_REQUIRED; - } + core::annotation::Input getInputRequirement() const override { + return core::annotation::Input::INPUT_REQUIRED; + } bool create_directory_; uint64_t batch_size_; @@ -128,5 +126,3 @@ REGISTER_RESOURCE(PutSFTP, "Sends FlowFiles to an SFTP Server"); } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/sftp/processors/SFTPProcessorBase.cpp b/extensions/sftp/processors/SFTPProcessorBase.cpp index d650865..ec95831 100644 --- a/extensions/sftp/processors/SFTPProcessorBase.cpp +++ b/extensions/sftp/processors/SFTPProcessorBase.cpp @@ -48,59 +48,74 @@ namespace nifi { namespace minifi { namespace processors { -core::Property SFTPProcessorBase::Hostname( - core::PropertyBuilder::createProperty("Hostname")->withDescription("The fully qualified hostname or IP address of the remote system") - ->isRequired(true)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::Port( - core::PropertyBuilder::createProperty("Port")->withDescription("The port that the remote system is listening on for file transfers") - ->isRequired(true)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::Username( - core::PropertyBuilder::createProperty("Username")->withDescription("Username") - ->isRequired(true)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::Password( - core::PropertyBuilder::createProperty("Password")->withDescription("Password for the user account") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::PrivateKeyPath( - core::PropertyBuilder::createProperty("Private Key Path")->withDescription("The fully qualified path to the Private Key file") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::PrivateKeyPassphrase( - core::PropertyBuilder::createProperty("Private Key Passphrase")->withDescription("Password for the private key") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::StrictHostKeyChecking( - core::PropertyBuilder::createProperty("Strict Host Key Checking")->withDescription("Indicates whether or not strict enforcement of hosts keys should be applied") - ->isRequired(true)->withDefaultValue<bool>(false)->build()); -core::Property SFTPProcessorBase::HostKeyFile( - core::PropertyBuilder::createProperty("Host Key File")->withDescription("If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used") - ->isRequired(false)->build()); -core::Property SFTPProcessorBase::ConnectionTimeout( - core::PropertyBuilder::createProperty("Connection Timeout")->withDescription("Amount of time to wait before timing out while creating a connection") - ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("30 sec")->build()); -core::Property SFTPProcessorBase::DataTimeout( - core::PropertyBuilder::createProperty("Data Timeout")->withDescription("When transferring a file between the local and remote system, this value specifies how long is allowed to elapse without any data being transferred between systems") - ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("30 sec")->build()); -core::Property SFTPProcessorBase::SendKeepaliveOnTimeout( - core::PropertyBuilder::createProperty("Send Keep Alive On Timeout")->withDescription("Indicates whether or not to send a single Keep Alive message when SSH socket times out") - ->isRequired(true)->withDefaultValue<bool>(true)->build()); -core::Property SFTPProcessorBase::ProxyType( - core::PropertyBuilder::createProperty("Proxy Type")->withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests. If set, it supersedes proxy settings configured per component. " - "Supported proxies: HTTP + AuthN, SOCKS + AuthN") - ->isRequired(false) - ->withAllowableValues<std::string>({PROXY_TYPE_DIRECT, - PROXY_TYPE_HTTP, - PROXY_TYPE_SOCKS}) - ->withDefaultValue(PROXY_TYPE_DIRECT)->build()); -core::Property SFTPProcessorBase::ProxyHost( - core::PropertyBuilder::createProperty("Proxy Host")->withDescription("The fully qualified hostname or IP address of the proxy server") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::ProxyPort( - core::PropertyBuilder::createProperty("Proxy Port")->withDescription("The port of the proxy server") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::HttpProxyUsername( - core::PropertyBuilder::createProperty("Http Proxy Username")->withDescription("Http Proxy Username") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); -core::Property SFTPProcessorBase::HttpProxyPassword( - core::PropertyBuilder::createProperty("Http Proxy Password")->withDescription("Http Proxy Password") - ->isRequired(false)->supportsExpressionLanguage(true)->build()); +core::Property SFTPProcessorBase::Hostname(core::PropertyBuilder::createProperty("Hostname") + ->withDescription("The fully qualified hostname or IP address of the remote system") + ->isRequired(true)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::Port(core::PropertyBuilder::createProperty("Port") + ->withDescription("The port that the remote system is listening on for file transfers") + ->isRequired(true)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::Username(core::PropertyBuilder::createProperty("Username") + ->withDescription("Username") + ->isRequired(true)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::Password(core::PropertyBuilder::createProperty("Password") + ->withDescription("Password for the user account") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::PrivateKeyPath(core::PropertyBuilder::createProperty("Private Key Path") + ->withDescription("The fully qualified path to the Private Key file") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::PrivateKeyPassphrase(core::PropertyBuilder::createProperty("Private Key Passphrase") + ->withDescription("Password for the private key") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::StrictHostKeyChecking(core::PropertyBuilder::createProperty("Strict Host Key Checking") + ->withDescription("Indicates whether or not strict enforcement of hosts keys should be applied") + ->isRequired(true)->withDefaultValue<bool>(false)->build()); + +core::Property SFTPProcessorBase::HostKeyFile(core::PropertyBuilder::createProperty("Host Key File") + ->withDescription("If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used") + ->isRequired(false)->build()); + +core::Property SFTPProcessorBase::ConnectionTimeout(core::PropertyBuilder::createProperty("Connection Timeout") + ->withDescription("Amount of time to wait before timing out while creating a connection") + ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("30 sec")->build()); + +core::Property SFTPProcessorBase::DataTimeout(core::PropertyBuilder::createProperty("Data Timeout") + ->withDescription("When transferring a file between the local and remote system, this value specifies how long is allowed to elapse without any data being transferred between systems") + ->isRequired(true)->withDefaultValue<core::TimePeriodValue>("30 sec")->build()); + +core::Property SFTPProcessorBase::SendKeepaliveOnTimeout(core::PropertyBuilder::createProperty("Send Keep Alive On Timeout") + ->withDescription("Indicates whether or not to send a single Keep Alive message when SSH socket times out") + ->isRequired(true)->withDefaultValue<bool>(true)->build()); + +core::Property SFTPProcessorBase::ProxyType(core::PropertyBuilder::createProperty("Proxy Type") + ->withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests. If set, it supersedes proxy settings configured per component. " + "Supported proxies: HTTP + AuthN, SOCKS + AuthN") + ->isRequired(false) + ->withAllowableValues<std::string>({PROXY_TYPE_DIRECT, + PROXY_TYPE_HTTP, + PROXY_TYPE_SOCKS}) + ->withDefaultValue(PROXY_TYPE_DIRECT)->build()); + +core::Property SFTPProcessorBase::ProxyHost(core::PropertyBuilder::createProperty("Proxy Host") + ->withDescription("The fully qualified hostname or IP address of the proxy server") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::ProxyPort(core::PropertyBuilder::createProperty("Proxy Port") + ->withDescription("The port of the proxy server") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::HttpProxyUsername(core::PropertyBuilder::createProperty("Http Proxy Username") + ->withDescription("Http Proxy Username") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); + +core::Property SFTPProcessorBase::HttpProxyPassword(core::PropertyBuilder::createProperty("Http Proxy Password") + ->withDescription("Http Proxy Password") + ->isRequired(false)->supportsExpressionLanguage(true)->build()); constexpr char const* SFTPProcessorBase::PROXY_TYPE_DIRECT; constexpr char const* SFTPProcessorBase::PROXY_TYPE_HTTP; diff --git a/extensions/sftp/processors/SFTPProcessorBase.h b/extensions/sftp/processors/SFTPProcessorBase.h index 7f5556b..e738589 100644 --- a/extensions/sftp/processors/SFTPProcessorBase.h +++ b/extensions/sftp/processors/SFTPProcessorBase.h @@ -16,8 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SFTP_PROCESSOR_BASE_H__ -#define __SFTP_PROCESSOR_BASE_H__ +#pragma once #include <memory> #include <string> @@ -25,16 +24,13 @@ #include <map> #include <mutex> #include <thread> +#include <set> -#include "utils/ByteArrayCallback.h" #include "FlowFileRecord.h" #include "core/Processor.h" #include "core/ProcessSession.h" -#include "core/Core.h" #include "core/Property.h" -#include "core/Resource.h" #include "controllers/SSLContextService.h" -#include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" #include "../client/SFTPClient.h" @@ -152,5 +148,3 @@ class SFTPProcessorBase : public core::Processor { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif diff --git a/extensions/sftp/tests/FetchSFTPTests.cpp b/extensions/sftp/tests/FetchSFTPTests.cpp index 14c33b5..ffdc434 100644 --- a/extensions/sftp/tests/FetchSFTPTests.cpp +++ b/extensions/sftp/tests/FetchSFTPTests.cpp @@ -142,7 +142,7 @@ class FetchSFTPTestsFixture { std::fstream file; std::stringstream ss; ss << src_dir << "/vfs/" << relative_path; - utils::file::FileUtils::create_dir(utils::file::FileUtils::get_parent_path(ss.str())); // TODO + utils::file::create_dir(utils::file::get_parent_path(ss.str())); file.open(ss.str(), std::ios::out); file << content; file.close(); diff --git a/extensions/sftp/tests/ListSFTPTests.cpp b/extensions/sftp/tests/ListSFTPTests.cpp index 1722e10..d59006b 100644 --- a/extensions/sftp/tests/ListSFTPTests.cpp +++ b/extensions/sftp/tests/ListSFTPTests.cpp @@ -58,8 +58,8 @@ class ListSFTPTestsFixture { public: - ListSFTPTestsFixture(const std::shared_ptr<minifi::Configure>& configuration = nullptr) - : src_dir(strdup("/var/tmp/sftps.XXXXXX")) { + explicit ListSFTPTestsFixture(const std::shared_ptr<minifi::Configure>& configuration = nullptr) + : src_dir(strdup("/var/tmp/sftps.XXXXXX")) { LogTestController::getInstance().setTrace<TestPlan>(); LogTestController::getInstance().setDebug<minifi::FlowController>(); LogTestController::getInstance().setDebug<minifi::SchedulingAgent>(); @@ -174,7 +174,7 @@ class ListSFTPTestsFixture { }; class PersistentListSFTPTestsFixture : public ListSFTPTestsFixture { -public: + public: PersistentListSFTPTestsFixture() : ListSFTPTestsFixture(std::make_shared<minifi::Configure>()) { } diff --git a/extensions/sftp/tests/PutSFTPTests.cpp b/extensions/sftp/tests/PutSFTPTests.cpp index fd1cba2..89df1aa 100644 --- a/extensions/sftp/tests/PutSFTPTests.cpp +++ b/extensions/sftp/tests/PutSFTPTests.cpp @@ -57,6 +57,8 @@ #include "processors/UpdateAttribute.h" #include "tools/SFTPTestServer.h" +constexpr const char* PUBLIC_KEY_AUTH_ERROR_MESSAGE = "Failed to authenticate with publickey, error: Unable to extract public key from private key file: Wrong passphrase or invalid/unrecognized private key file format"; // NOLINT(whitespace/line_length) + class PutSFTPTestsFixture { public: PutSFTPTestsFixture() @@ -270,8 +272,7 @@ TEST_CASE_METHOD(PutSFTPTestsFixture, "PutSFTP public key authentication bad pas std::string expected = minifi::Exception(minifi::PROCESS_SESSION_EXCEPTION, "Can not find the transfer relationship for the updated flow").what(); REQUIRE(0 == std::string(e.what()).compare(0, expected.size(), expected)); } - - REQUIRE(LogTestController::getInstance().contains("Failed to authenticate with publickey, error: Unable to extract public key from private key file: Wrong passphrase or invalid/unrecognized private key file format")); + REQUIRE(LogTestController::getInstance().contains(PUBLIC_KEY_AUTH_ERROR_MESSAGE)); REQUIRE(LogTestController::getInstance().contains("Could not authenticate with any available method")); } @@ -283,7 +284,7 @@ TEST_CASE_METHOD(PutSFTPTestsFixture, "PutSFTP public key authentication bad pas testController.runSession(plan, true); - REQUIRE(LogTestController::getInstance().contains("Failed to authenticate with publickey, error: Unable to extract public key from private key file: Wrong passphrase or invalid/unrecognized private key file format")); + REQUIRE(LogTestController::getInstance().contains(PUBLIC_KEY_AUTH_ERROR_MESSAGE)); REQUIRE(LogTestController::getInstance().contains("Successfully authenticated with password")); testFile("nifi_test/tstFile.ext", "tempFile"); } diff --git a/extensions/sftp/tests/tools/SFTPTestServer.cpp b/extensions/sftp/tests/tools/SFTPTestServer.cpp index 5a1c15d..6ab788a 100644 --- a/extensions/sftp/tests/tools/SFTPTestServer.cpp +++ b/extensions/sftp/tests/tools/SFTPTestServer.cpp @@ -21,8 +21,7 @@ #include <thread> #include <exception> -#ifdef WIN32 -#else +#ifndef WIN32 #include <signal.h> #include <unistd.h> #include <fcntl.h> @@ -35,15 +34,14 @@ #include "utils/file/FileUtils.h" SFTPTestServer::SFTPTestServer(const std::string& working_directory, - const std::string& host_key_file /*= "resources/host.pem"*/, - const std::string& jar_path /*= "tools/sftp-test-server/target/SFTPTestServer-1.0.0.jar"*/) - : logger_(logging::LoggerFactory<SFTPTestServer>::getLogger()) - , working_directory_(working_directory) - , started_(false) - , port_(0U) -#ifdef WIN32 -#else - , server_pid_(-1) + const std::string& host_key_file /*= "resources/host.pem"*/, + const std::string& jar_path /*= "tools/sftp-test-server/target/SFTPTestServer-1.0.0.jar"*/) + : logger_(logging::LoggerFactory<SFTPTestServer>::getLogger()), + working_directory_(working_directory), + started_(false), + port_(0U) +#ifndef WIN32 + , server_pid_(-1) #endif { auto executable_dir = utils::file::FileUtils::get_executable_dir(); diff --git a/extensions/sftp/tests/tools/SFTPTestServer.h b/extensions/sftp/tests/tools/SFTPTestServer.h index 5ede833..9acf377 100644 --- a/extensions/sftp/tests/tools/SFTPTestServer.h +++ b/extensions/sftp/tests/tools/SFTPTestServer.h @@ -15,13 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SFTP_TEST_SERVER_H__ -#define __SFTP_TEST_SERVER_H__ +#pragma once #include <cstdint> #include <string> -#ifdef WIN32 -#else +#include <memory> + +#ifndef WIN32 #include <unistd.h> #include <sys/types.h> #endif @@ -31,7 +31,6 @@ class SFTPTestServer { public: - SFTPTestServer(const std::string& working_directory, const std::string& host_key_file = "resources/host.pem", const std::string& jar_path = "tools/sftp-test-server/target/SFTPTestServer-1.0.0.jar"); @@ -42,7 +41,6 @@ class SFTPTestServer { uint16_t getPort(); private: - std::shared_ptr<logging::Logger> logger_; std::string host_key_file_; @@ -51,10 +49,7 @@ class SFTPTestServer { bool started_; std::string port_file_path_; uint16_t port_; -#ifdef WIN32 -#else +#ifndef WIN32 pid_t server_pid_; #endif }; - -#endif
