This is an automated email from the ASF dual-hosted git repository.
phrocker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/master by this push:
new 1250d3c MINIFICPP-856 Introduce AWS SDK to the MiNiFi Cmake build
1250d3c is described below
commit 1250d3c8f98d9df38d745f2d170d191e2bf677f4
Author: Jeremy Dyer <[email protected]>
AuthorDate: Mon Apr 22 10:22:35 2019 -0400
MINIFICPP-856 Introduce AWS SDK to the MiNiFi Cmake build
Updated documentation as pointed out by @phrocker
This closes #562.
Signed-off-by: Marc Parisi <[email protected]>
---
CMakeLists.txt | 5 +
CONTROLLERS.md | 39 ++++++++
README.md | 1 +
bootstrap.sh | 2 +
bstrp_functions.sh | 2 +
extensions/aws/AWSLoader.cpp | 27 ++++++
extensions/aws/AWSLoader.h | 70 ++++++++++++++
extensions/aws/CMakeLists.txt | 67 +++++++++++++
.../controllerservices/AWSCredentialsService.cpp | 57 +++++++++++
.../aws/controllerservices/AWSCredentialsService.h | 104 +++++++++++++++++++++
extensions/aws/s3/S3.h | 73 +++++++++++++++
11 files changed, 447 insertions(+)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3f74bfa..c3ab96b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -553,6 +553,11 @@ if (ENABLE_TENSORFLOW)
createExtension(TENSORFLOW-EXTENSIONS "TENSORFLOW EXTENSIONS" "This
enables TensorFlow support" "extensions/tensorflow"
"${TEST_DIR}/tensorflow-tests")
endif()
+## AWS Extentions
+if (ENABLE_AWS)
+ createExtension(AWS-EXTENSIONS "AWS EXTENSIONS" "This enables AWS
support" "extensions/aws" )
+endif()
+
## Bustache/template extensions
option(ENABLE_BUSTACHE "Enables Bustache (ApplyTemplate) support." OFF)
diff --git a/CONTROLLERS.md b/CONTROLLERS.md
new file mode 100644
index 0000000..f1482f8
--- /dev/null
+++ b/CONTROLLERS.md
@@ -0,0 +1,39 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Controller Services
+
+## Table of Contents
+
+- [AWSCredentialsService](#awsCredentialsService)
+
+## AWSCredentialsService
+
+### Description
+
+Manages the Amazon Web Services (AWS) ```Access Key``` and ```Secret Access
Key``` for a AWS account. This allows for multiple
+AWS credential services to be defined. This also allows for multiple AWS
related processors to reference this single
+controller service so that AWS credentials can be managed and controlled in a
central location.
+
+### Properties
+
+In the list below, the names of required properties appear in bold. Any other
+properties (not in bold) are considered optional. The table also indicates any
+default values, and whether a property supports the NiFi Expression Language.
+
+| Name | Default Value | Allowable Values | Expression Language Supported? |
Description |
+| - | - | - | - |
+| **Access Key** | | | Yes | Specifies the AWS Access Key |
+| **Secret Key** | | | Yes | Specifies the AWS Secret Key |
\ No newline at end of file
diff --git a/README.md b/README.md
index 18ee178..898be3c 100644
--- a/README.md
+++ b/README.md
@@ -70,6 +70,7 @@ Through JNI extensions you can run NiFi processors using
NARs. The JNI extension
| Extension Set | Processors | CMAKE Flag |
| ------------- |:-------------| :-----|
| Archive Extensions |
[ApplyTemplate](PROCESSORS.md#applytemplate)<br/>[CompressContent](PROCESSORS.md#compresscontent)<br/>[ManipulateArchive](PROCESSORS.md#manipulatearchive)<br/>[MergeContent](PROCESSORS.md#mergecontent)<br/>[FocusArchiveEntry](PROCESSORS.md#focusarchiveentry)<br/>[UnfocusArchiveEntry](PROCESSORS.md#unfocusarchiveentry)
| -DBUILD_LIBARCHIVE=ON |
+| AWS | [AWSCredentialsService](CONTROLLERS.md#awsCredentialsService) |
-DENABLE_AWS=ON |
| CURL | [InvokeHTTP](PROCESSORS.md#invokehttp) | -DDISABLE_CURL=ON |
| GPS | GetGPS | -DENABLE_GPS=ON |
| Kafka | [PublishKafka](PROCESSORS.md#publishkafka) |
-DENABLE_LIBRDKAFKA=ON |
diff --git a/bootstrap.sh b/bootstrap.sh
index df6eb65..b85b18d 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -264,6 +264,8 @@ add_dependency USB_ENABLED "libpng"
add_disabled_option GPS_ENABLED ${FALSE} "ENABLE_GPS"
add_dependency GPS_ENABLED "gpsd"
+add_disabled_option AWS_ENABLED ${TRUE} "ENABLE_AWS"
+
add_disabled_option KAFKA_ENABLED ${FALSE} "ENABLE_LIBRDKAFKA" "3.4.0"
add_disabled_option MQTT_ENABLED ${FALSE} "ENABLE_MQTT"
diff --git a/bstrp_functions.sh b/bstrp_functions.sh
index aecffe6..2a5bfaf 100755
--- a/bstrp_functions.sh
+++ b/bstrp_functions.sh
@@ -259,6 +259,7 @@ show_supported_features() {
echo "M. SQLite Support ..............$(print_feature_status SQLITE_ENABLED)"
echo "N. Python Support ..............$(print_feature_status PYTHON_ENABLED)"
echo "O. COAP Support ................$(print_feature_status COAP_ENABLED)"
+ echo "V. AWS Support .................$(print_feature_status AWS_ENABLED)"
echo "****************************************"
echo " Build Options."
echo "****************************************"
@@ -293,6 +294,7 @@ read_feature_options(){
k) ToggleFeature BUSTACHE_ENABLED ;;
l) ToggleFeature MQTT_ENABLED ;;
m) ToggleFeature SQLITE_ENABLED ;;
+ v) ToggleFeature AWS_ENABLED ;;
n) if [ "$USE_SHARED_LIBS" = "${TRUE}" ]; then
ToggleFeature PYTHON_ENABLED
else
diff --git a/extensions/aws/AWSLoader.cpp b/extensions/aws/AWSLoader.cpp
new file mode 100644
index 0000000..652339f
--- /dev/null
+++ b/extensions/aws/AWSLoader.cpp
@@ -0,0 +1,27 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "AWSLoader.h"
+
+bool AWSObjectFactory::added =
core::FlowConfiguration::add_static_func("createAWSFactory");
+extern "C" {
+
+void *createAWSFactory(void) {
+ return new AWSObjectFactory();
+}
+
+}
diff --git a/extensions/aws/AWSLoader.h b/extensions/aws/AWSLoader.h
new file mode 100644
index 0000000..ab868e0
--- /dev/null
+++ b/extensions/aws/AWSLoader.h
@@ -0,0 +1,70 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_AWSLOADER_H
+#define NIFI_MINIFI_CPP_AWSLOADER_H
+
+#include "core/ClassLoader.h"
+#include "utils/StringUtils.h"
+#include "controllerservices/AWSCredentialsService.h"
+
+class AWSObjectFactory : public core::ObjectFactory {
+
+ public:
+ AWSObjectFactory() {
+
+ }
+
+ /**
+ * Gets the name of the object.
+ * @return class name of processor
+ */
+ virtual std::string getName() override{
+ return "AWSObjectFactory";
+ }
+
+ virtual std::string getClassName() override{
+ return "AWSObjectFactory";
+ }
+
+ /**
+ * Gets the class name for the object
+ * @return class name for the processor.
+ */
+ virtual std::vector<std::string> getClassNames() override{
+ std::vector<std::string> class_names;
+ class_names.push_back("AWSCredentialsService");
+ return class_names;
+ }
+
+ virtual std::unique_ptr<ObjectFactory> assign(const std::string &class_name)
override{
+ if (utils::StringUtils::equalsIgnoreCase(class_name,
"AWSCredentialsService")) {
+ return std::unique_ptr<ObjectFactory>(new
core::DefautObjectFactory<minifi::aws::controllers::AWSCredentialsService>());
+ } else {
+ return nullptr;
+ }
+ }
+
+ static bool added;
+};
+
+extern "C" {
+DLL_EXPORT void *createAWSFactory(void);
+}
+
+#endif //NIFI_MINIFI_CPP_AWSLOADER_H
diff --git a/extensions/aws/CMakeLists.txt b/extensions/aws/CMakeLists.txt
new file mode 100644
index 0000000..a80f862
--- /dev/null
+++ b/extensions/aws/CMakeLists.txt
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
+include_directories(controllerservices)
+
+file(GLOB SOURCES "*.cpp" "s3/*.cpp" "controllerservices/*.cpp")
+
+set(BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp")
+set(BYPRODUCT "${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-install/")
+
+# AWS SDK 1.7.89
+ExternalProject_Add(
+ awssdk-external
+ GIT_REPOSITORY "https://github.com/aws/aws-sdk-cpp.git";
+ GIT_TAG "09b65deba03cfbef9a1e5d5986aa4de71bc03cd8"
+ EXCLUDE_FROM_ALL TRUE
+ INSTALL_DIR ${BYPRODUCT}
+ CMAKE_ARGS -DBUILD_ONLY=s3
+ -DENABLE_TESTING=OFF
+ -DBUILD_SHARED_LIBS=ON
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo
+ -DCMAKE_INSTALL_PREFIX=${BYPRODUCT}
+)
+
+add_library(awssdklib STATIC IMPORTED)
+set_target_properties(awssdklib PROPERTIES IMPORTED_LOCATION "${BYPRODUCT}")
+
+set(AWSSDK_FOUND "YES" CACHE STRING "" FORCE)
+set(AWSSDK_INCLUDE_DIRS "${BYPRODUCT}/include" CACHE STRING "" FORCE)
+set(AWSSDK_LIBRARIES awssdklib CACHE STRING "" FORCE)
+set(AWSSDK_LIBRARY awssdklib CACHE STRING "" FORCE)
+
+include_directories(${AWSSDK_INCLUDE_DIRS})
+add_library(minifi-aws STATIC ${SOURCES})
+
+if (APPLE)
+ target_link_libraries (minifi-aws
${BYPRODUCT}lib/libaws-c-event-stream.dylib)
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-c-common.dylib)
+ target_link_libraries (minifi-aws
${BYPRODUCT}lib/libaws-cpp-sdk-core.dylib)
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-cpp-sdk-s3.dylib)
+else ()
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-c-event-stream.so)
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-c-common.so)
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-cpp-sdk-core.so)
+ target_link_libraries (minifi-aws ${BYPRODUCT}lib/libaws-cpp-sdk-s3.so)
+endif ()
+
+add_dependencies(minifi-aws awssdk-external)
+SET (AWS-EXTENSION minifi-aws PARENT_SCOPE)
+register_extension(minifi-aws)
\ No newline at end of file
diff --git a/extensions/aws/controllerservices/AWSCredentialsService.cpp
b/extensions/aws/controllerservices/AWSCredentialsService.cpp
new file mode 100644
index 0000000..4e958b3
--- /dev/null
+++ b/extensions/aws/controllerservices/AWSCredentialsService.cpp
@@ -0,0 +1,57 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AWSCredentialsService.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace aws {
+namespace controllers {
+
+core::Property AWSCredentialsService::AccessKey(
+ core::PropertyBuilder::createProperty("Access
Key")->withDescription("Specifies the AWS Access
Key.")->isRequired(true)->supportsExpressionLanguage(true)->build());
+
+core::Property AWSCredentialsService::SecretKey(
+ core::PropertyBuilder::createProperty("Secret
Key")->withDescription("Specifies the AWS Secret
Key.")->isRequired(true)->supportsExpressionLanguage(true)->build());
+
+void AWSCredentialsService::initialize() {
+ std::set<core::Property> supportedProperties;
+ supportedProperties.insert(AccessKey);
+ supportedProperties.insert(SecretKey);
+ setSupportedProperties(supportedProperties);
+}
+
+void AWSCredentialsService::onEnable() {
+ getProperty(AccessKey.getName(), s3Ack);
+ getProperty(SecretKey.getName(), s3Secret);
+
+ Aws::String awsS3Secret = s3Secret.c_str();
+ Aws::String awsS3Ack = s3Ack.c_str();
+
+ Aws::Auth::AWSCredentials creds(awsS3Ack, awsS3Secret);
+ awsCredentials = creds;
+}
+
+} /* namespace controllers */
+} /* namespace AWS */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
\ No newline at end of file
diff --git a/extensions/aws/controllerservices/AWSCredentialsService.h
b/extensions/aws/controllerservices/AWSCredentialsService.h
new file mode 100644
index 0000000..a9cb69d
--- /dev/null
+++ b/extensions/aws/controllerservices/AWSCredentialsService.h
@@ -0,0 +1,104 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_AWSCREDENTIALSCONTROLLERSERVICE_H
+#define NIFI_MINIFI_CPP_AWSCREDENTIALSCONTROLLERSERVICE_H
+
+#include <string>
+#include <iostream>
+#include <memory>
+#include "core/Resource.h"
+#include "utils/StringUtils.h"
+#include "io/validation.h"
+#include "core/controller/ControllerService.h"
+#include "core/logging/LoggerConfiguration.h"
+#include "core/FlowConfiguration.h"
+
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/Bucket.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentials.h>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace aws {
+namespace controllers {
+
+class AWSCredentialsService : public core::controller::ControllerService {
+
+ public:
+
+ static core::Property AccessKey;
+ static core::Property SecretKey;
+
+ explicit AWSCredentialsService(const std::string &name, const std::string
&id)
+ : ControllerService(name, id),
+ logger_(logging::LoggerFactory<AWSCredentialsService>::getLogger()) {
+ }
+
+ explicit AWSCredentialsService(const std::string &name, utils::Identifier
uuid = utils::Identifier())
+ : ControllerService(name, uuid),
+ logger_(logging::LoggerFactory<AWSCredentialsService>::getLogger()) {
+ }
+
+ explicit AWSCredentialsService(const std::string &name, const
std::shared_ptr<Configure> &configuration)
+ : ControllerService(name),
+ logger_(logging::LoggerFactory<AWSCredentialsService>::getLogger()) {
+
+ }
+
+ virtual void initialize() override;
+
+ virtual void yield() override {
+
+ };
+
+ virtual bool isWorkAvailable() override {
+ return false;
+ };
+
+ virtual bool isRunning() override {
+ return getState() == core::controller::ControllerServiceState::ENABLED;
+ }
+
+ virtual void onEnable() override;
+
+ Aws::Auth::AWSCredentials getAWSCredentials() {
+ return awsCredentials;
+ }
+
+ private:
+ std::string s3Ack, s3Secret;
+ Aws::Auth::AWSCredentials awsCredentials;
+ Aws::Client::ClientConfiguration client_config_;
+ std::shared_ptr<logging::Logger> logger_;
+};
+
+REGISTER_RESOURCE(AWSCredentialsService, "AWS Credentials Management Service");
+
+} /* namespace controllers */
+} /* namespace AWS */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_AWSCREDENTIALSCONTROLLERSERVICE_H
diff --git a/extensions/aws/s3/S3.h b/extensions/aws/s3/S3.h
new file mode 100644
index 0000000..a78d748
--- /dev/null
+++ b/extensions/aws/s3/S3.h
@@ -0,0 +1,73 @@
+/**
+ * @file S3.h
+ * GetGPS class declaration
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __S3_H__
+#define __S3_H__
+
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/Bucket.h>
+
+#include <aws/core/utils/memory/stl/AWSString.h>
+#include <aws/core/utils/logging/DefaultLogSystem.h>
+#include <aws/core/utils/logging/AWSLogging.h>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+
+class S3Initializer{
+ public:
+ S3Initializer(){
+ Aws::InitAPI(options);
+ Aws::Utils::Logging::InitializeAWSLogging(
+ Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(
+ "RunUnitTests", Aws::Utils::Logging::LogLevel::Trace, "aws_sdk_"));
+ }
+
+ ~S3Initializer(){
+ Aws::ShutdownAPI(options);
+ }
+
+ private:
+ Aws::SDKOptions options;
+};
+
+class S3Singleton{
+ public:
+ static S3Initializer *get(){
+ static S3Initializer init;
+ return &init;
+ }
+ private:
+ S3Singleton(){
+
+ }
+};
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif
\ No newline at end of file
