markap14 commented on a change in pull request #4730: URL: https://github.com/apache/nifi/pull/4730#discussion_r561919864
########## File path: nifi-external/nifi-kafka-connect/nifi-kafka-connector/src/main/java/org/apache/nifi/kafka/connect/StatelessNiFiSourceTask.java ########## @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.kafka.connect; + +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.RetriableException; +import org.apache.kafka.connect.header.ConnectHeaders; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.stateless.flow.DataflowTrigger; +import org.apache.nifi.stateless.flow.StatelessDataflow; +import org.apache.nifi.stateless.flow.TriggerResult; +import org.apache.nifi.util.FormatUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; + +public class StatelessNiFiSourceTask extends SourceTask { + public static final String STATE_MAP_KEY = "task.index"; + private static final Logger logger = LoggerFactory.getLogger(StatelessNiFiSourceTask.class); + + private StatelessDataflow dataflow; + private String outputPortName; + private String topicName; + private String topicNameAttribute; + private TriggerResult triggerResult; + private String keyAttributeName; + private Pattern headerAttributeNamePattern; + private long timeoutMillis; + private String dataflowName; + private long failureYieldExpiration = 0L; + + private final Map<String, String> clusterStatePartitionMap = Collections.singletonMap(STATE_MAP_KEY, "CLUSTER"); + private Map<String, String> localStatePartitionMap = new HashMap<>(); + + private Map<String, ?> previousBatchComponentStates = Collections.emptyMap(); + + private final AtomicLong unacknowledgedRecords = new AtomicLong(0L); + + @Override + public String version() { + return StatelessKafkaConnectorUtil.getVersion(); + } + + @Override + public void start(final Map<String, String> properties) { + logger.info("Starting Source Task with properties {}", StatelessKafkaConnectorUtil.getLoggableProperties(properties)); + + final String timeout = properties.getOrDefault(StatelessKafkaConnectorUtil.DATAFLOW_TIMEOUT, StatelessKafkaConnectorUtil.DEFAULT_DATAFLOW_TIMEOUT); + timeoutMillis = (long) FormatUtils.getPreciseTimeDuration(timeout, TimeUnit.MILLISECONDS); + + topicName = properties.get(StatelessNiFiSourceConnector.TOPIC_NAME); + topicNameAttribute = properties.get(StatelessNiFiSourceConnector.TOPIC_NAME_ATTRIBUTE); + keyAttributeName = properties.get(StatelessNiFiSourceConnector.KEY_ATTRIBUTE); + + if (topicName == null && topicNameAttribute == null) { + throw new ConfigException("Either the topic.name or topic.name.attribute configuration must be specified"); + } + + final String headerRegex = properties.get(StatelessNiFiSourceConnector.HEADER_REGEX); + headerAttributeNamePattern = headerRegex == null ? null : Pattern.compile(headerRegex); + + dataflow = StatelessKafkaConnectorUtil.createDataflow(properties); + + // Determine the name of the Output Port to retrieve data from + dataflowName = properties.get(StatelessKafkaConnectorUtil.DATAFLOW_NAME); + outputPortName = properties.get(StatelessNiFiSourceConnector.OUTPUT_PORT_NAME); + if (outputPortName == null) { + final Set<String> outputPorts = dataflow.getOutputPortNames(); + if (outputPorts.isEmpty()) { + throw new ConfigException("The dataflow specified for <" + dataflowName + "> does not have an Output Port at the root level. Dataflows used for a Kafka Connect Source Task " + + "must have at least one Output Port at the root level."); + } + + if (outputPorts.size() > 1) { + throw new ConfigException("The dataflow specified for <" + dataflowName + "> has multiple Output Ports at the root level (" + outputPorts.toString() + + "). The " + StatelessNiFiSourceConnector.OUTPUT_PORT_NAME + " property must be set to indicate which of these Ports Kafka records should be retrieved from."); + } + + outputPortName = outputPorts.iterator().next(); + } + + final String taskIndex = properties.get(STATE_MAP_KEY); Review comment: Yes. So in NiFi, processors can store either 'local' state or 'cluster-wide' state. Typically local state would be used if tailing a file on the local file system or something like that. In that case, it would map to the task id. If you want from 3 tasks to 2, that's fine. The state/progress would be lost so there may be some data duplication. But tailing a file in a local file system wouldn't really be a great use case for connect since the task could be potentially started elsewhere, etc. For the most part local state won't be used, though. Cluster-wide state would be used if gathering data from S3 or a GCS bucket, for example. In that case, you'd likely only use a single task due to the fact that the protocol itself doesn't offer queuing semantics so it's difficult to scale. But for cases where cluster-wide state is used by the Processor, the state is instead stored here using the `clusterStatePartitionMap`. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
