simonbence commented on a change in pull request #4714: URL: https://github.com/apache/nifi/pull/4714#discussion_r564617844
########## File path: nifi-nar-bundles/nifi-splunk-bundle/nifi-splunk-processors/src/main/java/org/apache/nifi/processors/splunk/QuerySplunkIndexingStatus.java ########## @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.splunk; + +import com.splunk.RequestMessage; +import com.splunk.ResponseMessage; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.ReadsAttribute; +import org.apache.nifi.annotation.behavior.ReadsAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.dto.splunk.EventIndexStatusRequest; +import org.apache.nifi.dto.splunk.EventIndexStatusResponse; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@Tags({"splunk", "logs", "http", "acknowledgement"}) +@CapabilityDescription("Queries Splunk server in order to acquire the status of indexing acknowledgement.") +@ReadsAttributes({ + @ReadsAttribute(attribute = "splunk.acknowledgement.id", description = "The indexing acknowledgement id provided by Splunk."), + @ReadsAttribute(attribute = "splunk.responded.at", description = "The time of the response of put request for Splunk.")}) +@SeeAlso(PutSplunkHTTP.class) +public class QuerySplunkIndexingStatus extends SplunkAPICall { + private static final String ENDPOINT = "/services/collector/ack"; + + static final Relationship RELATIONSHIP_ACKNOWLEDGED = new Relationship.Builder() + .name("success") + .description("A FlowFile is transferred to this relationship when the acknowledgement was successful.") + .build(); + + static final Relationship RELATIONSHIP_UNACKNOWLEDGED = new Relationship.Builder() + .name("unacknowledged") + .description( + "A FlowFile is transferred to this relationship when the acknowledgement was not successful." + + "This can happen when the acknowledgement did not happened within the time period set for Maximum Waiting Time. " + + "FlowFiles with acknowledgement id unknown for the Splunk server will be transferred to this relationship after the Maximum Waiting Time is reached.") + .build(); + + static final Relationship RELATIONSHIP_UNDETERMINED = new Relationship.Builder() + .name("undetermined") + .description( + "A FlowFile is transferred to this relationship when the acknowledgement state is not determined. " + + "FlowFiles transferred to this relationship might be penalized! " + + "This happens when Splunk returns with HTTP 200 but with false response for the acknowledgement id in the flow file attribute.") Review comment: Splunk responses with a boolean value for this query. The only case it returns with "true" value, is the _first_ call after the acknowledgement is done. (Given it's called soon enough and is not moved out from the internal buffers of Splunk) In every other case, including failed acknowledgement, early poll, unknown id and further polls on acknowledged items, it will return "false", without further details. Thus, the "false" response is perfectly valid. In order to avoid hitting the external resource (Splunk) continuously, I do keep the "question" open (transfering to undetermined), until it's answered by the Splunk properly or times out, and penalise the FF. Assuming it is looped back to the processor, in a later time, it should be tried again. Practically this means, we polled too early and we do not want to try again with the given id for a while. As the processor works with batches of acknowledgement requests this is not equal to decreasing the frequency of the triggering. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
