[
https://issues.apache.org/jira/browse/NIFI-1767?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15310777#comment-15310777
]
ASF GitHub Bot commented on NIFI-1767:
--------------------------------------
Github user JPercivall commented on a diff in the pull request:
https://github.com/apache/nifi/pull/349#discussion_r65411771
--- Diff:
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/iot/GetAWSIoT.java
---
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.iot;
+
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.aws.iot.util.IoTMessage;
+import org.apache.nifi.processors.aws.iot.util.MqttWebSocketAsyncClient;
+import org.eclipse.paho.client.mqttv3.MqttException;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.LinkedList;
+import java.util.HashMap;
+import java.util.Map;
+
+@Tags({"Amazon", "AWS", "IOT", "MQTT", "Websockets", "Get", "Subscribe",
"Receive"})
+@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN)
+@CapabilityDescription("Subscribes to and receives messages from
MQTT-topic(s) of AWS IoT." +
+ "The processor keeps open a WebSocket connection and will
automatically renew the " +
+ "connection to overcome Amazon's service limit on maximum connection
duration. Depending on " +
+ "your set up QoS the processor will miss some messages (QoS=0) or
receives messages twice (QoS=1) " +
+ "while reconnecting to AWS IoT WebSocket endpoint. We strongly
recommend you to make use of " +
+ "processor isolation as concurrent subscriptions to an MQTT topic
result in multiple message receiptions.")
+@SeeAlso({ GetAWSIoTShadow.class })
+@WritesAttributes({
+ @WritesAttribute(attribute = "aws.iot.mqtt.endpoint", description
= "AWS endpoint this message was received from."),
+ @WritesAttribute(attribute = "aws.iot.mqtt.topic", description =
"MQTT topic this message was received from."),
+ @WritesAttribute(attribute = "aws.iot.mqtt.client", description =
"MQTT client which received the message."),
+ @WritesAttribute(attribute = "aws.iot.mqtt.qos", description =
"Underlying MQTT quality-of-service.")
+})
+public class GetAWSIoT extends AbstractAWSIoTProcessor {
+
+ public static final List<PropertyDescriptor> properties =
Collections.unmodifiableList(
+ Arrays.asList(
+ PROP_QOS,
+ PROP_TOPIC,
+ PROP_ENDPOINT,
+ PROP_KEEPALIVE,
+ PROP_CLIENT,
+ AWS_CREDENTIALS_PROVIDER_SERVICE,
+ REGION));
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Collections.singleton(REL_SUCCESS);
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+ // init to build up mqtt connection over web-sockets
+ init(context);
+ if (mqttClient != null && mqttClient.isConnected()) {
+ try {
+ // subscribe to topic with configured qos in order to
start receiving messages
+ mqttClient.subscribe(awsTopic, awsQos);
+ } catch (MqttException e) {
+ getLogger().error("Error while subscribing to topic " +
awsTopic + " with client-id " + mqttClient.getClientId() + " caused by " +
e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final
ProcessSession session) throws ProcessException {
+ final List messageList = new LinkedList();
+ // check if connection is about to terminate
+ if (isConnectionAboutToExpire()) {
+ MqttWebSocketAsyncClient _mqttClient = null;
+ try {
+ // before subscribing to the topic with new connection
first unsubscribe
+ // old connection from same topic if subscription is set
to QoS 0
+ if (awsQos == 0) mqttClient.unsubscribe(awsTopic);
+ // establish a second connection
+ _mqttClient = connect(context);
+ // now subscribe to topic with new connection
+ _mqttClient.subscribe(awsTopic, awsQos);
+ // between re-subscription and disconnect from old
connection
+ // QoS=0 subscription eventually lose some messages
+ // QoS=1 subscription eventually receive some messages
twice
+ // now terminate old connection
+ mqttClient.disconnect();
+ } catch (MqttException e) {
+ getLogger().error("Error while renewing connection with
client " + mqttClient.getClientId() + " caused by " + e.getMessage());
+ } finally {
+ // grab messages left over from old connection
+ mqttClient.getAwsQueuedMqttMessages().drainTo(messageList);
+ // now set the new connection as the default connection
+ if (_mqttClient != null) mqttClient = _mqttClient;
+ }
+ } else {
+ // grab messages which queued up since last run
+ mqttClient.getAwsQueuedMqttMessages().drainTo(messageList);
--- End diff --
What is the benefit of draining the queue into the local messageList? One
distinct problem I see is after the queue is drained into the local variable
`messageList` and an exception occurs, all messages remaining in the queue will
be lost.
What I would suggest doing is to peek at the top of the queue and then
remove it only after it has been transferred successfully, like in
ConsumeMQTT[1]
[1]
https://github.com/apache/nifi/blob/f47af1ce8336c9305916f00738976f3505b01b0b/nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/ConsumeMQTT.java#L268
> AWS IoT processors
> ------------------
>
> Key: NIFI-1767
> URL: https://issues.apache.org/jira/browse/NIFI-1767
> Project: Apache NiFi
> Issue Type: New Feature
> Components: Extensions
> Reporter: Kay Lerch
> Attachments: 20160413_apache-nifi-aws-iot-pull-request_lerchkay.pdf
>
>
> Four new processors to communicate with Amazon’s managed device gateway
> service AWS IoT.
> h5.Use cases
> * Consume reported states from a fleet of things managed and secured on
> Amazon’s gateway service
> * Propagate desired states to a fleet of things managed and secured on
> Amazon’s gateway service
> * Intercept M2M communication
> * Hybrid IoT solutions: brings together a managed device gateway in the cloud
> and onpremise data-consumers and -providers.
> h4.GetIOTMqtt:
> Opens up a connection to an AWS-account-specific websocket endpoint in order
> to subscribe to any of the MQTT topics belonging to a registered thing in AWS
> IoT.
> h4.PutIOTMqtt
> Opens up a connection to an AWS-account-specific websocket endpoint in order
> to publish messages to any of the MQTT topics belonging to a registered thing
> in AWS IoT.
> h4.GetIOTShadow
> In AWS IoT a physical thing is represented with its last reported state by
> the so-called thing shadow. This processor reads out the current state of a
> shadow (persisted as JSON) by requesting the managed API of AWS IoT.
> h4.PutIOTShadow
> In AWS IoT a physical thing is represented with its last reported state by
> the so-called thing shadow. This processor updates the current state of a
> shadow (persisted as JSON) by requesting the managed API of AWS IoT. An
> update to a shadow lets AWS IoT propagate changes to the MQTT topics of the
> thing.
> h5.Known issues:
> * It was hard for me to write appropriate integration tests since the MQTT
> processors work with durable websocket-connections which are kind of tough to
> test. With your help I would love to do a better job on testing and hand it
> in later on. All of the processors were tested in a live-scenario which ran
> over a longer period of time. Didn’t observe any issue.
> * I got rid of all the properties for the deprecated
> AWSCredentialProviderService and only made use of
> AWSCredentialsProviderControllerService. If both are still necessary for
> backward-compatibilities sake I would add the deprecated feature.
> Refers to Pull Request 349: https://github.com/apache/nifi/pull/349
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
