[GitHub] [nifi] turcsanyip commented on a diff in pull request #6225: NIFI-10251 Add v5 protocol support for existing MQTT processors
turcsanyip commented on code in PR #6225:
URL: https://github.com/apache/nifi/pull/6225#discussion_r954994656
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/common/AbstractMQTTProcessor.java:
##
@@ -31,89 +31,89 @@
import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.ssl.SSLContextService;
-import org.eclipse.paho.client.mqttv3.IMqttClient;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
-import org.eclipse.paho.client.mqttv3.MqttException;
-import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import static org.apache.commons.lang3.EnumUtils.isValidEnumIgnoreCase;
+import static org.apache.commons.lang3.StringUtils.EMPTY;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_CLEAN_SESSION_FALSE;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_CLEAN_SESSION_TRUE;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_MQTT_VERSION_310;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_MQTT_VERSION_311;
+import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_MQTT_VERSION_500;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_MQTT_VERSION_AUTO;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_QOS_0;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_QOS_1;
import static
org.apache.nifi.processors.mqtt.common.MqttConstants.ALLOWABLE_VALUE_QOS_2;
public abstract class AbstractMQTTProcessor extends
AbstractSessionFactoryProcessor {
-public static int DISCONNECT_TIMEOUT = 5000;
+private static final long DEFAULT_SESSION_EXPIRY_INTERVAL_IN_SECONDS =
3600;
Review Comment:
I feel the 3600 seconds (1 hour) default value a bit strict. It means that
if NiFi is down for 1 hour, the consume processor will loose messages. I would
set it to 24 hours or so (in v3 it was infinite).
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/ConsumeMQTT.java:
##
@@ -402,42 +398,27 @@ private void initializeClient(ProcessContext context) {
// non-null but not connected, so we need to handle each case and only
create a new client when it is null
try {
if (mqttClient == null) {
-logger.debug("Creating client");
-mqttClient = createMqttClient(broker, clientID, persistence);
+mqttClient = createMqttClient();
mqttClient.setCallback(this);
}
if (!mqttClient.isConnected()) {
-logger.debug("Connecting client");
-mqttClient.connect(connOpts);
+mqttClient.connect();
mqttClient.subscribe(topicPrefix + topicFilter, qos);
}
-} catch (MqttException e) {
-logger.error("Connection to {} lost (or was never connected) and
connection failed. Yielding processor", new Object[]{broker}, e);
+} catch (Exception e) {
+logger.error("Connection to {} lost (or was never connected) and
connection failed. Yielding processor", new
Object[]{clientProperties.getBroker()}, e);
+mqttClient = null; // prevent stucked processor when subscribe
fails
Review Comment:
I recommend to execute the same logic as in case of `OnStopped` in order to
prevent resource leaking due to clients which fail to subscribe but already
connected successfully (and remains on the heap until it gets garbage
collected).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@nifi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [nifi] turcsanyip commented on a diff in pull request #6225: NIFI-10251 Add v5 protocol support for existing MQTT processors
turcsanyip commented on code in PR #6225:
URL: https://github.com/apache/nifi/pull/6225#discussion_r953145681
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/adapters/HiveMqV5ClientAdapter.java:
##
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.mqtt.adapters;
+
+import com.hivemq.client.mqtt.datatypes.MqttQos;
+import com.hivemq.client.mqtt.mqtt5.Mqtt5BlockingClient;
+import com.hivemq.client.mqtt.mqtt5.message.connect.Mqtt5Connect;
+import com.hivemq.client.mqtt.mqtt5.message.connect.Mqtt5ConnectBuilder;
+import com.hivemq.client.mqtt.mqtt5.message.subscribe.suback.Mqtt5SubAck;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processors.mqtt.common.MqttCallback;
+import org.apache.nifi.processors.mqtt.common.MqttClient;
+import org.apache.nifi.processors.mqtt.common.MqttConnectionProperties;
+import org.apache.nifi.processors.mqtt.common.MqttException;
+import org.apache.nifi.processors.mqtt.common.ReceivedMqttMessage;
+import org.apache.nifi.processors.mqtt.common.StandardMqttMessage;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+
+public class HiveMqV5ClientAdapter implements MqttClient {
+
+private final Mqtt5BlockingClient mqtt5BlockingClient;
+private final ComponentLog logger;
+
+private MqttCallback callback;
+
+public HiveMqV5ClientAdapter(Mqtt5BlockingClient mqtt5BlockingClient,
ComponentLog logger) {
+this.mqtt5BlockingClient = mqtt5BlockingClient;
+this.logger = logger;
+}
+
+@Override
+public boolean isConnected() {
+return mqtt5BlockingClient.getState().isConnected();
+}
+
+@Override
+public void connect(MqttConnectionProperties connectionProperties) {
+logger.debug("Connecting to broker");
+
+final Mqtt5ConnectBuilder connectBuilder = Mqtt5Connect.builder()
+.keepAlive(connectionProperties.getKeepAliveInterval());
+
+final boolean cleanSession = connectionProperties.isCleanSession();
+connectBuilder.cleanStart(cleanSession);
+if (!cleanSession) {
+
connectBuilder.sessionExpiryInterval(connectionProperties.getSessionExpiryInterval());
+}
+
+final String lastWillTopic = connectionProperties.getLastWillTopic();
+if (lastWillTopic != null) {
+connectBuilder.willPublish()
+.topic(lastWillTopic)
+
.payload(connectionProperties.getLastWillMessage().getBytes())
+.retain(connectionProperties.getLastWillRetain())
+
.qos(MqttQos.fromCode(connectionProperties.getLastWillQOS()))
+.applyWillPublish();
+}
+
+final String username = connectionProperties.getUsername();
+final String password = connectionProperties.getPassword();
+if (username != null && password != null) {
+connectBuilder.simpleAuth()
+.username(connectionProperties.getUsername())
+.password(password.getBytes(StandardCharsets.UTF_8))
+.applySimpleAuth();
+}
+
+final Mqtt5Connect mqtt5Connect = connectBuilder.build();
+mqtt5BlockingClient.connect(mqtt5Connect);
+}
+
+@Override
+public void disconnect(long disconnectTimeout) {
+logger.debug("Disconnecting client");
+// Currently it is not possible to set timeout for disconnect with
HiveMQ Client. (Only connect timeout exists.)
+mqtt5BlockingClient.disconnect();
+}
+
+@Override
+public void close() {
+// there is no paho's close equivalent in hivemq client
+}
+
+@Override
+public void publish(String topic, StandardMqttMessage message) {
+logger.debug("Publishing message to {} with QoS: {}", topic,
message.getQos());
+
+mqtt5BlockingClient.publishWith()
+.topic(topic)
+.payload(message.getPayload())
+.retain(message.isRetained())
+
[GitHub] [nifi] turcsanyip commented on a diff in pull request #6225: NIFI-10251 Add v5 protocol support for existing MQTT processors
turcsanyip commented on code in PR #6225:
URL: https://github.com/apache/nifi/pull/6225#discussion_r953140061
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/common/AbstractMQTTProcessor.java:
##
@@ -289,88 +288,35 @@ public Collection customValidate(final
ValidationContext valid
return results;
}
-public static Properties transformSSLContextService(SSLContextService
sslContextService){
-Properties properties = new Properties();
-if (sslContextService.getSslAlgorithm() != null) {
-properties.setProperty("com.ibm.ssl.protocol",
sslContextService.getSslAlgorithm());
-}
-if (sslContextService.getKeyStoreFile() != null) {
-properties.setProperty("com.ibm.ssl.keyStore",
sslContextService.getKeyStoreFile());
-}
-if (sslContextService.getKeyStorePassword() != null) {
-properties.setProperty("com.ibm.ssl.keyStorePassword",
sslContextService.getKeyStorePassword());
-}
-if (sslContextService.getKeyStoreType() != null) {
-properties.setProperty("com.ibm.ssl.keyStoreType",
sslContextService.getKeyStoreType());
-}
-if (sslContextService.getTrustStoreFile() != null) {
-properties.setProperty("com.ibm.ssl.trustStore",
sslContextService.getTrustStoreFile());
-}
-if (sslContextService.getTrustStorePassword() != null) {
-properties.setProperty("com.ibm.ssl.trustStorePassword",
sslContextService.getTrustStorePassword());
-}
-if (sslContextService.getTrustStoreType() != null) {
-properties.setProperty("com.ibm.ssl.trustStoreType",
sslContextService.getTrustStoreType());
-}
-return properties;
-}
-
-protected void onScheduled(final ProcessContext context){
-broker =
context.getProperty(PROP_BROKER_URI).evaluateAttributeExpressions().getValue();
-brokerUri = broker.endsWith("/") ? broker : broker + "/";
-clientID =
context.getProperty(PROP_CLIENTID).evaluateAttributeExpressions().getValue();
-
-if (clientID == null) {
-clientID = UUID.randomUUID().toString();
-}
-
-connOpts = new MqttConnectOptions();
-
connOpts.setCleanSession(context.getProperty(PROP_CLEAN_SESSION).asBoolean());
-
connOpts.setKeepAliveInterval(context.getProperty(PROP_KEEP_ALIVE_INTERVAL).asInteger());
-
connOpts.setMqttVersion(context.getProperty(PROP_MQTT_VERSION).asInteger());
-
connOpts.setConnectionTimeout(context.getProperty(PROP_CONN_TIMEOUT).asInteger());
-
-PropertyValue sslProp = context.getProperty(PROP_SSL_CONTEXT_SERVICE);
-if (sslProp.isSet()) {
-Properties sslProps =
transformSSLContextService((SSLContextService) sslProp.asControllerService());
-connOpts.setSSLProperties(sslProps);
-}
-
-PropertyValue lastWillTopicProp =
context.getProperty(PROP_LAST_WILL_TOPIC);
-if (lastWillTopicProp.isSet()){
-String lastWillMessage =
context.getProperty(PROP_LAST_WILL_MESSAGE).getValue();
-PropertyValue lastWillRetain =
context.getProperty(PROP_LAST_WILL_RETAIN);
-Integer lastWillQOS =
context.getProperty(PROP_LAST_WILL_QOS).asInteger();
-connOpts.setWill(lastWillTopicProp.getValue(),
lastWillMessage.getBytes(), lastWillQOS, lastWillRetain.isSet() ?
lastWillRetain.asBoolean() : false);
-}
-
-
-PropertyValue usernameProp = context.getProperty(PROP_USERNAME);
-if(usernameProp.isSet()) {
-
connOpts.setUserName(usernameProp.evaluateAttributeExpressions().getValue());
-
connOpts.setPassword(context.getProperty(PROP_PASSWORD).getValue().toCharArray());
-}
+protected void onScheduled(final ProcessContext context) {
+clientProperties = getMqttClientProperties(context);
+connectionProperties = getMqttConnectionProperties(context);
}
protected void onStopped() {
-try {
-logger.info("Disconnecting client");
-mqttClient.disconnect(DISCONNECT_TIMEOUT);
-} catch(MqttException me) {
-logger.error("Error disconnecting MQTT client due to {}", new
Object[]{me.getMessage()}, me);
-}
+// Since client is created in the onTrigger method it can happen that
it never will be created because of an initialization error.
+// We are preventing additional nullPtrException here, but the clean
solution would be to create the client in the onScheduled method.
+if (mqttClient != null) {
+try {
+logger.info("Disconnecting client");
+mqttClient.disconnect(DISCONNECT_TIMEOUT);
+} catch (MqttException me) {
+logger.error("Error disconnecting MQTT client due to {}", new
Object[]{me.getMessage()}, me);
+}
+
+
[GitHub] [nifi] turcsanyip commented on a diff in pull request #6225: NIFI-10251 Add v5 protocol support for existing MQTT processors
turcsanyip commented on code in PR #6225:
URL: https://github.com/apache/nifi/pull/6225#discussion_r950100231
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/common/MqttConstants.java:
##
@@ -67,14 +69,45 @@ public class MqttConstants {
*/
public static final AllowableValue ALLOWABLE_VALUE_MQTT_VERSION_AUTO =
new
AllowableValue(String.valueOf(MqttConnectOptions.MQTT_VERSION_DEFAULT),
-"AUTO",
+"v3 AUTO",
"Start with v3.1.1 and fallback to v3.1.0 if not supported
by a broker");
+public static final AllowableValue ALLOWABLE_VALUE_MQTT_VERSION_500 =
+new
AllowableValue(String.valueOf(MqttVersion.MQTT_VERSION_5_0.getNumericValue()),
+"v5.0");
+
public static final AllowableValue ALLOWABLE_VALUE_MQTT_VERSION_311 =
-new
AllowableValue(String.valueOf(MqttConnectOptions.MQTT_VERSION_3_1_1),
+new
AllowableValue(String.valueOf(MqttVersion.MQTT_VERSION_3_1_1.getNumericValue()),
"v3.1.1");
public static final AllowableValue ALLOWABLE_VALUE_MQTT_VERSION_310 =
-new
AllowableValue(String.valueOf(MqttConnectOptions.MQTT_VERSION_3_1),
+new
AllowableValue(String.valueOf(MqttVersion.MQTT_VERSION_3_1.getNumericValue()),
"v3.1.0");
+
+public enum MqttVersion {
+MQTT_VERSION_3_1(3),
+MQTT_VERSION_3_1_1(4),
+MQTT_VERSION_5_0(5);
+
+private final int numericValue;
Review Comment:
`versionCode` may be more descriptive.
Its type could be `String` because it is always be used as `String`.
##
nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/adapters/HiveMqV5ClientAdapter.java:
##
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.mqtt.adapters;
+
+import com.hivemq.client.mqtt.datatypes.MqttQos;
+import com.hivemq.client.mqtt.mqtt5.Mqtt5BlockingClient;
+import com.hivemq.client.mqtt.mqtt5.Mqtt5Client;
+import com.hivemq.client.mqtt.mqtt5.message.connect.Mqtt5Connect;
+import com.hivemq.client.mqtt.mqtt5.message.connect.Mqtt5ConnectBuilder;
+import org.apache.nifi.processors.mqtt.common.MqttConnectionProperties;
+import org.apache.nifi.processors.mqtt.common.NifiMqttCallback;
+import org.apache.nifi.processors.mqtt.common.NifiMqttClient;
+import org.apache.nifi.processors.mqtt.common.NifiMqttException;
+import org.apache.nifi.processors.mqtt.common.NifiMqttMessage;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class HiveMqV5ClientAdapter implements NifiMqttClient {
+
+private final Mqtt5Client mqtt5Client;
+
+private NifiMqttCallback callback;
+
+public HiveMqV5ClientAdapter(Mqtt5BlockingClient mqtt5BlockingClient) {
+this.mqtt5Client = mqtt5BlockingClient;
+}
+
+@Override
+public boolean isConnected() {
+return mqtt5Client.getState().isConnected();
+}
+
+@Override
+public void connect(MqttConnectionProperties connectionProperties) throws
NifiMqttException {
+final Mqtt5ConnectBuilder connectBuilder = Mqtt5Connect.builder()
+.keepAlive(connectionProperties.getKeepAliveInterval());
+
+final boolean cleanSession = connectionProperties.isCleanSession();
+connectBuilder.cleanStart(cleanSession);
+if (!cleanSession) {
+
connectBuilder.sessionExpiryInterval(connectionProperties.getSessionExpiryInterval());
+}
+
+final String lastWillTopic = connectionProperties.getLastWillTopic();
+if (lastWillTopic != null) {
+
