[GitHub] nifi pull request #1125: NIFI-2872 Create PutCloudwatchMetric Processor
Github user vegaed commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1125#discussion_r83200380
--- Diff:
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/cloudwatch/PutCloudWatchMetric.java
---
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.cloudwatch;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient;
+import com.amazonaws.services.cloudwatch.model.MetricDatum;
+import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import
org.apache.nifi.processors.aws.AbstractAWSCredentialsProviderProcessor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
+@Tags({"amazon", "aws", "cloudwatch", "metrics", "put", "publish"})
+@CapabilityDescription("Publishes metrics to Amazon CloudWatch")
+public class PutCloudWatchMetric extends
AbstractAWSCredentialsProviderProcessor {
+
+public static final Set relationships =
Collections.unmodifiableSet(
+new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)));
+
+private static final Validator DOUBLE_VALIDATOR = new Validator() {
+@Override
+public ValidationResult validate(String subject, String input,
ValidationContext context) {
+if (context.isExpressionLanguageSupported(subject) &&
context.isExpressionLanguagePresent(input)) {
+return (new
ValidationResult.Builder()).subject(subject).input(input).explanation("Expression
Language Present").valid(true).build();
+} else {
+String reason = null;
+
+try {
+Double.parseDouble(input);
+} catch (NumberFormatException var6) {
+reason = "not a valid Double";
+}
+
+return (new
ValidationResult.Builder()).subject(subject).input(input).explanation(reason).valid(reason
== null).build();
+}
+}
+};
+
+public static final PropertyDescriptor NAMESPACE = new
PropertyDescriptor.Builder()
+.name("Namespace")
+.displayName("Namespace")
+.description("The namespace for the metric data for
CloudWatch")
+.required(true)
+.expressionLanguageSupported(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor METRIC_NAME = new
PropertyDescriptor.Builder()
+.name("MetricName")
+.displayName("MetricName")
+.description("The name of the metric")
+.expressionLanguageSupported(true)
+.required(
[jira] [Commented] (NIFI-2872) Create PutCloudwatchMetric Processor
[
https://issues.apache.org/jira/browse/NIFI-2872?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571722#comment-15571722
]
ASF GitHub Bot commented on NIFI-2872:
--
Github user vegaed commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1125#discussion_r83200380
--- Diff:
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/cloudwatch/PutCloudWatchMetric.java
---
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.cloudwatch;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient;
+import com.amazonaws.services.cloudwatch.model.MetricDatum;
+import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import
org.apache.nifi.processors.aws.AbstractAWSCredentialsProviderProcessor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
+@Tags({"amazon", "aws", "cloudwatch", "metrics", "put", "publish"})
+@CapabilityDescription("Publishes metrics to Amazon CloudWatch")
+public class PutCloudWatchMetric extends
AbstractAWSCredentialsProviderProcessor {
+
+public static final Set relationships =
Collections.unmodifiableSet(
+new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)));
+
+private static final Validator DOUBLE_VALIDATOR = new Validator() {
+@Override
+public ValidationResult validate(String subject, String input,
ValidationContext context) {
+if (context.isExpressionLanguageSupported(subject) &&
context.isExpressionLanguagePresent(input)) {
+return (new
ValidationResult.Builder()).subject(subject).input(input).explanation("Expression
Language Present").valid(true).build();
+} else {
+String reason = null;
+
+try {
+Double.parseDouble(input);
+} catch (NumberFormatException var6) {
+reason = "not a valid Double";
+}
+
+return (new
ValidationResult.Builder()).subject(subject).input(input).explanation(reason).valid(reason
== null).build();
+}
+}
+};
+
+public static final PropertyDescriptor NAMESPACE = new
PropertyDescriptor.Builder()
+.name("Namespace")
+.displayName("Namespace")
+.description("The namespace for the metric data for
CloudWatch")
+.required(true)
+.expressionLanguageSupported(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor METRIC_NAME
[GitHub] nifi issue #1125: NIFI-2872 Create PutCloudwatchMetric Processor
Github user vegaed commented on the issue: https://github.com/apache/nifi/pull/1125 @jvwing Thanks for all the great suggestions and code. I moved the try up in order to catch those types of parsing exceptions so now they will route to failure. Good catch on the integration tests Yes I agree unit testing is good. I was modeling off of S3 and SQS and didn't see any there, but I see now that dynamo and kinesis packages are being both unit and integration tested. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-2872) Create PutCloudwatchMetric Processor
[ https://issues.apache.org/jira/browse/NIFI-2872?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571748#comment-15571748 ] ASF GitHub Bot commented on NIFI-2872: -- Github user vegaed commented on the issue: https://github.com/apache/nifi/pull/1125 @jvwing Thanks for all the great suggestions and code. I moved the try up in order to catch those types of parsing exceptions so now they will route to failure. Good catch on the integration tests Yes I agree unit testing is good. I was modeling off of S3 and SQS and didn't see any there, but I see now that dynamo and kinesis packages are being both unit and integration tested. > Create PutCloudwatchMetric Processor > > > Key: NIFI-2872 > URL: https://issues.apache.org/jira/browse/NIFI-2872 > Project: Apache NiFi > Issue Type: Task > Components: Core Framework >Affects Versions: 1.1.0 >Reporter: Edgardo Vega >Priority: Minor > > Create a processor for put metrics into aws cloudwatch. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi issue #1122: NIFI-2863: S2S to allow cluster URL more leniently
Github user mcgilman commented on the issue: https://github.com/apache/nifi/pull/1122 Reviewing... --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-2863) A remote process group pointed to a host without the trailing "/nifi" will fail with mis-leading bulletins
[
https://issues.apache.org/jira/browse/NIFI-2863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571756#comment-15571756
]
ASF GitHub Bot commented on NIFI-2863:
--
Github user mcgilman commented on the issue:
https://github.com/apache/nifi/pull/1122
Reviewing...
> A remote process group pointed to a host without the trailing "/nifi" will
> fail with mis-leading bulletins
> --
>
> Key: NIFI-2863
> URL: https://issues.apache.org/jira/browse/NIFI-2863
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.0.0
>Reporter: Joseph Percivall
>Assignee: Koji Kawamura
> Fix For: 1.1.0
>
>
> To replicate:
> 1: Set up NiFi instance on port 8080 and remote port 8081 (unsecure S2S)
> 2: create input port
> 3: create RPG pointing to "http://localhost:8080";
> This RPG will correctly get the instance name, listing of ports and port
> status but when transmission is enabled and a flowfile is queued to be sent
> the following error is generated:
> "RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]"
> Looking at the logs there is this message:
> 2016-10-04 14:11:34,298 WARN [Timer-Driven Process Thread-2]
> o.a.n.r.util.SiteToSiteRestApiClient Failed to parse Json, response=
>
>
>
>
>
> NiFi
>
> />
> type="text/css" />
> href="/nifi/assets/font-awesome/css/font-awesome.min.css" type="text/css" />
> type="text/css" />
> type="text/css" />
>
>
>
>
> Did you mean: /nifi
>
> You may have mistyped...
>
>
>
> 2016-10-04 14:11:34,298 ERROR [Timer-Driven Process Thread-2]
> o.a.nifi.remote.StandardRemoteGroupPort
> RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]
> This should either be fixed (to allow without "/nifi") or explicitly
> validated.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2500) Allow request buffer to be configurable on HandleHTTPRequest processor
[ https://issues.apache.org/jira/browse/NIFI-2500?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2500: -- Component/s: (was: Core Framework) Extensions > Allow request buffer to be configurable on HandleHTTPRequest processor > -- > > Key: NIFI-2500 > URL: https://issues.apache.org/jira/browse/NIFI-2500 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Affects Versions: 0.6.1 >Reporter: Matthew Clarke >Assignee: Oleg Zhurakousky >Priority: Critical > Fix For: 1.1.0 > > > The request buffer for the HandleHTTPRequest buffer is hard coded to 50. For > environments where bursts of requests can come in that exceed that threshold, > the processor will trigger Service Unavailable responses. Users should be > able to increase that buffer to meet their dataflow needs similar to how the > ConsumeMQTT processor works. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2500) Allow request buffer to be configurable on HandleHTTPRequest processor
[ https://issues.apache.org/jira/browse/NIFI-2500?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2500: -- Fix Version/s: (was: 1.1.0) > Allow request buffer to be configurable on HandleHTTPRequest processor > -- > > Key: NIFI-2500 > URL: https://issues.apache.org/jira/browse/NIFI-2500 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Affects Versions: 0.6.1 >Reporter: Matthew Clarke >Assignee: Oleg Zhurakousky >Priority: Critical > > The request buffer for the HandleHTTPRequest buffer is hard coded to 50. For > environments where bursts of requests can come in that exceed that threshold, > the processor will trigger Service Unavailable responses. Users should be > able to increase that buffer to meet their dataflow needs similar to how the > ConsumeMQTT processor works. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1991) Zero Master Cluster: Modifying a disconnected node causes other nodes to not be able to join cluster
[ https://issues.apache.org/jira/browse/NIFI-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-1991: -- Priority: Major (was: Blocker) > Zero Master Cluster: Modifying a disconnected node causes other nodes to not > be able to join cluster > - > > Key: NIFI-1991 > URL: https://issues.apache.org/jira/browse/NIFI-1991 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Andrew Lim > > I created a zero master cluster with 3 nodes. All three were up and running. > I stopped all 3 instances and then brought up just Node1. > In the Node1 UI, I made some edits to the flow. > I attempted but was not able to bring up Node2 and Node3 to join the cluster. > Here is what I saw in the logs: > 2016-06-09 14:35:47,051 WARN [main] org.apache.nifi.web.server.JettyServer > Failed to start web server... shutting down. > java.lang.Exception: Unable to load flow due to: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:753) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.(NiFi.java:137) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.main(NiFi.java:227) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > Caused by: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:501) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:744) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 2 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Failed to > connect node to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:862) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:497) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 3 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Proposed > configuration is not inheritable by the flow controller because of flow > differences: Found difference in Flows: > Local Fingerprint contains additional configuration from Cluster Fingerprint: > eba77dac-32ad-356b-8a82-f669d046aa21eba77dac-32ad-356b-8a82-f669d046aa21org.apache.nifi.processors.standard.LogAttributeNO_VALUEAttributes > to IgnoreNO_VALUEAttributes to LogNO_VALUELog prefixNO_VALUEs > at > org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:219) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1329) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:75) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:668) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:839) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 4 common frames omitted > 2016-06-09 14:35:47,052 INFO [Thread-1] org.apache.nifi.NiFi Initiating > shutdown of Jetty web server... -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-1991) Zero Master Cluster: Modifying a disconnected node causes other nodes to not be able to join cluster
[ https://issues.apache.org/jira/browse/NIFI-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571772#comment-15571772 ] Joseph Witt commented on NIFI-1991: --- based on the discussion i've reduced the priority and removed the fix version > Zero Master Cluster: Modifying a disconnected node causes other nodes to not > be able to join cluster > - > > Key: NIFI-1991 > URL: https://issues.apache.org/jira/browse/NIFI-1991 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Andrew Lim > > I created a zero master cluster with 3 nodes. All three were up and running. > I stopped all 3 instances and then brought up just Node1. > In the Node1 UI, I made some edits to the flow. > I attempted but was not able to bring up Node2 and Node3 to join the cluster. > Here is what I saw in the logs: > 2016-06-09 14:35:47,051 WARN [main] org.apache.nifi.web.server.JettyServer > Failed to start web server... shutting down. > java.lang.Exception: Unable to load flow due to: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:753) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.(NiFi.java:137) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.main(NiFi.java:227) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > Caused by: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:501) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:744) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 2 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Failed to > connect node to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:862) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:497) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 3 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Proposed > configuration is not inheritable by the flow controller because of flow > differences: Found difference in Flows: > Local Fingerprint contains additional configuration from Cluster Fingerprint: > eba77dac-32ad-356b-8a82-f669d046aa21eba77dac-32ad-356b-8a82-f669d046aa21org.apache.nifi.processors.standard.LogAttributeNO_VALUEAttributes > to IgnoreNO_VALUEAttributes to LogNO_VALUELog prefixNO_VALUEs > at > org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:219) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1329) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:75) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:668) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:839) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 4 common frames omitted > 2016-06-09 14:35:47,052 INFO [Thread-1] org.apache.nifi.NiFi Initiating > shutdown of Jetty web server... -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1991) Zero Master Cluster: Modifying a disconnected node causes other nodes to not be able to join cluster
[ https://issues.apache.org/jira/browse/NIFI-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-1991: -- Fix Version/s: (was: 1.1.0) > Zero Master Cluster: Modifying a disconnected node causes other nodes to not > be able to join cluster > - > > Key: NIFI-1991 > URL: https://issues.apache.org/jira/browse/NIFI-1991 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Andrew Lim >Priority: Blocker > > I created a zero master cluster with 3 nodes. All three were up and running. > I stopped all 3 instances and then brought up just Node1. > In the Node1 UI, I made some edits to the flow. > I attempted but was not able to bring up Node2 and Node3 to join the cluster. > Here is what I saw in the logs: > 2016-06-09 14:35:47,051 WARN [main] org.apache.nifi.web.server.JettyServer > Failed to start web server... shutting down. > java.lang.Exception: Unable to load flow due to: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:753) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.(NiFi.java:137) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.main(NiFi.java:227) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > Caused by: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:501) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:744) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 2 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Failed to > connect node to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:862) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:497) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 3 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Proposed > configuration is not inheritable by the flow controller because of flow > differences: Found difference in Flows: > Local Fingerprint contains additional configuration from Cluster Fingerprint: > eba77dac-32ad-356b-8a82-f669d046aa21eba77dac-32ad-356b-8a82-f669d046aa21org.apache.nifi.processors.standard.LogAttributeNO_VALUEAttributes > to IgnoreNO_VALUEAttributes to LogNO_VALUELog prefixNO_VALUEs > at > org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:219) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1329) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:75) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:668) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:839) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 4 common frames omitted > 2016-06-09 14:35:47,052 INFO [Thread-1] org.apache.nifi.NiFi Initiating > shutdown of Jetty web server... -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1500) NiFi requires too much write permissions to bootstrap
[
https://issues.apache.org/jira/browse/NIFI-1500?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joseph Witt updated NIFI-1500:
--
Fix Version/s: (was: 1.1.0)
> NiFi requires too much write permissions to bootstrap
> -
>
> Key: NIFI-1500
> URL: https://issues.apache.org/jira/browse/NIFI-1500
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 0.5.0
>Reporter: Andre
>
> As part of NIFI-1461 it was observed that NiFi boot strapping and start up
> process require access that may be described as excessive.
> The following access requirements should be reviewed (all paths are relative
> to NiFi's installation path):
> h4. -Issue 1 - {{bin/nifi.sh}} requires write access to {{bin}}-
> -Access is used to write {{nifi.pid}}-
> h4. Issue 2 - NiFi requires write access to {{lib}} crashing otherwise
> {code}
> 2016-02-11 13:00:35,497 INFO [main] org.apache.nifi.NiFi Launching NiFi...
> 2016-02-11 13:00:35,524 INFO [main] org.apache.nifi.BootstrapListener Started
> Bootstrap Listener, Listening for incoming requests on port 52097
> 2016-02-11 13:00:35,545 INFO [main] org.apache.nifi.BootstrapListener
> Successfully initiated communication with Bootstrap
> 2016-02-11 13:00:35,555 WARN [main] org.apache.nifi.nar.NarUnpacker Unable to
> load NAR library bundles due to java.io.IOException:
> /opt/nifi/nifi-0.5.0-SNAPSHOT/./lib directory does not have read/write
> privilege Will proceed without loading any further Nar bundles
> 2016-02-11 13:00:35,559 ERROR [main] org.apache.nifi.NiFi Failure to launch
> NiFi due to java.lang.IllegalStateException: Unable to find the framework NAR
> ClassLoader.
> java.lang.IllegalStateException: Unable to find the framework NAR ClassLoader.
> at org.apache.nifi.NiFi.(NiFi.java:116)
> ~[nifi-runtime-0.5.0-SNAPSHOT.jar:0.5.0-SNAPSHOT]
> at org.apache.nifi.NiFi.main(NiFi.java:227)
> ~[nifi-runtime-0.5.0-SNAPSHOT.jar:0.5.0-SNAPSHOT]
> 2016-02-11 13:00:35,561 INFO [Thread-1] org.apache.nifi.NiFi Initiating
> shutdown of Jetty web server...
> 2016-02-11 13:00:35,561 INFO [Thread-1] org.apache.nifi.NiFi Jetty web server
> shutdown completed (nicely or otherwise).
> {code}
> h4. Issue 3 - NiFi requires write access to {{conf}}
> Access is used by the process to save {{flows.xml.gz}} and to create
> {{templates}} and {{archives}} upon first start.
> h4. Issue 4 - NiFi requires write access to {{.}}
> Application uses access upon first start to create the following directories:
> - \*_repositories;
> - logs;
> - state;
> - work;
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2002) Name property for funnels
[ https://issues.apache.org/jira/browse/NIFI-2002?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2002: -- Fix Version/s: (was: 1.1.0) > Name property for funnels > - > > Key: NIFI-2002 > URL: https://issues.apache.org/jira/browse/NIFI-2002 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Reporter: n h > Labels: Core, Funnel > > Adding "name" property for funnels is very handy when using API to > manage flows (non-graphical batch clients), -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1963) Cluster - Flow inheritability
[ https://issues.apache.org/jira/browse/NIFI-1963?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-1963: -- Fix Version/s: (was: 1.1.0) > Cluster - Flow inheritability > - > > Key: NIFI-1963 > URL: https://issues.apache.org/jira/browse/NIFI-1963 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Matt Gilman > > I disconnected a node and changed the penalty duration. The attempted to > reconnect the node. The node joined successfully with an inconsistent penalty > duration. > Need to reconsider which fields need to belong in the flow fingerprint to > ensure consistent configuration throughout the cluster. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1135) For Provenance Query, bring back Event Summaries instead of the Events themselves
[ https://issues.apache.org/jira/browse/NIFI-1135?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-1135: -- Fix Version/s: (was: 1.1.0) > For Provenance Query, bring back Event Summaries instead of the Events > themselves > - > > Key: NIFI-1135 > URL: https://issues.apache.org/jira/browse/NIFI-1135 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework, Core UI >Reporter: Mark Payne >Assignee: Mark Payne > > Currently, when we query Provenance, we pull back up to 1000 events. These > are full Provenance Events with attributes, etc. If the query takes a long > time, we will request those objects that already have matched the query many > times. This amounts to a great deal of heap being used and sending back very > large JSON objects (10+ MB is not uncommon and it could potentially be far > worse). > We should instead use a ProvenanceEventSummary object. This object should > contain just the info shown in the results table and the pointer to the > actual event in the Provenance Store. This allows us to return the queries > much faster, store less data in the heap, and provide less data back to the > end user with virtually the same experience. > The one place that this would differ in UX is when the user clicks the "info" > button to view the entire provenance event, we would have to pull the event > back from the server, rather than already having that in memory. > We should consider storing all of the fields in the results table in Lucene > to provide faster results. Otherwise, we could still get potentially better > results with the current approach if we just ensure that the first fields > that we store are those in the results table. This allows us to read just a > small portion of the event from file and deserializing just a small amount of > data before moving on to the next event. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi issue #1080: NIFI-766 Added icon on connection when backpressure is ena...
Github user mcgilman commented on the issue: https://github.com/apache/nifi/pull/1080 Can someone else finish up the review of this PR? I helped contribute to this PR and would prefer someone else's eyes on. Thanks! --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-766) UI should indicate when backpressure is configured for a Connection
[ https://issues.apache.org/jira/browse/NIFI-766?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571841#comment-15571841 ] ASF GitHub Bot commented on NIFI-766: - Github user mcgilman commented on the issue: https://github.com/apache/nifi/pull/1080 Can someone else finish up the review of this PR? I helped contribute to this PR and would prefer someone else's eyes on. Thanks! > UI should indicate when backpressure is configured for a Connection > --- > > Key: NIFI-766 > URL: https://issues.apache.org/jira/browse/NIFI-766 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework, Core UI >Reporter: Mark Payne >Assignee: Pierre Villard > Fix For: 1.1.0 > > Attachments: backpressure.png, backpressure_and_expiration.png, > normal.png > > > It is sometimes unclear why a Processor is not running, if it is due to > backpressure. Recommend we add an icon to the Connection label to indicate > that backpressure is configured. If backpressure is "applied" (i.e., the > backpressure threshold has been reached), that icon should be highlighted > somehow. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2052) Search results
[ https://issues.apache.org/jira/browse/NIFI-2052?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2052: -- Affects Version/s: 1.0.0 > Search results > -- > > Key: NIFI-2052 > URL: https://issues.apache.org/jira/browse/NIFI-2052 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Scott Aslan > Attachments: nifi-search-results.png > > > New icons and styles to be applied to the search results container. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2052) Search results
[ https://issues.apache.org/jira/browse/NIFI-2052?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2052: -- Fix Version/s: (was: 1.1.0) > Search results > -- > > Key: NIFI-2052 > URL: https://issues.apache.org/jira/browse/NIFI-2052 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Scott Aslan > Attachments: nifi-search-results.png > > > New icons and styles to be applied to the search results container. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-1135) For Provenance Query, bring back Event Summaries instead of the Events themselves
[ https://issues.apache.org/jira/browse/NIFI-1135?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-1135: -- Affects Version/s: 1.0.0 > For Provenance Query, bring back Event Summaries instead of the Events > themselves > - > > Key: NIFI-1135 > URL: https://issues.apache.org/jira/browse/NIFI-1135 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework, Core UI >Affects Versions: 1.0.0 >Reporter: Mark Payne >Assignee: Mark Payne > > Currently, when we query Provenance, we pull back up to 1000 events. These > are full Provenance Events with attributes, etc. If the query takes a long > time, we will request those objects that already have matched the query many > times. This amounts to a great deal of heap being used and sending back very > large JSON objects (10+ MB is not uncommon and it could potentially be far > worse). > We should instead use a ProvenanceEventSummary object. This object should > contain just the info shown in the results table and the pointer to the > actual event in the Provenance Store. This allows us to return the queries > much faster, store less data in the heap, and provide less data back to the > end user with virtually the same experience. > The one place that this would differ in UX is when the user clicks the "info" > button to view the entire provenance event, we would have to pull the event > back from the server, rather than already having that in memory. > We should consider storing all of the fields in the results table in Lucene > to provide faster results. Otherwise, we could still get potentially better > results with the current approach if we just ensure that the first fields > that we store are those in the results table. This allows us to read just a > small portion of the event from file and deserializing just a small amount of > data before moving on to the next event. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2195) Improve sophistication of how nodes detect and handle differences between local and cluster flows
[ https://issues.apache.org/jira/browse/NIFI-2195?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2195: -- Affects Version/s: 1.0.0 > Improve sophistication of how nodes detect and handle differences between > local and cluster flows > - > > Key: NIFI-2195 > URL: https://issues.apache.org/jira/browse/NIFI-2195 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > Currently, when a node joins a cluster, it uses the flow's "Fingerprint" to > determine whether or not its flow matches the cluster flow. If not, it will > not join the cluster. There are two main drivers for this. First, we want to > ensure that we don't overwrite the flow with a completely different flow > (though we want to allow for minor differences between the flow, such as the > location of a processor on the graph). Secondly, we want to ensure that there > is no data loss. > We can improve the sophistication of this and allow a node to join the > cluster if there are no 'substantive' differences. It is critical that a node > not inherit a flow that is missing some Connection, if that Connection has > data queued up, as doing so would result in data loss. > We should be smarter, though, about which things really matter. For instance, > if the name of a Processor is different, the node should simply inherit the > name of the Processor from the cluster, instead of refusing to join. > If any changes to the flow are made when joining a cluster, though, we should > ensure that we back up the flow before changing it. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2078) State management for processors whose states are managed externally
[ https://issues.apache.org/jira/browse/NIFI-2078?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2078: -- Fix Version/s: (was: 1.1.0) > State management for processors whose states are managed externally > --- > > Key: NIFI-2078 > URL: https://issues.apache.org/jira/browse/NIFI-2078 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > Inherently by the nature of a given processor it may involve state managed by > itself (using nifi state management), or can be managed by some external > service it interacts with (kafka's offset), and theoretically some might have > both going on. With the new state management, we're giving users a way to > reset state managed by nifi for a given processor. But it doesnt apply to > those processors who have external state. > we should consider offering a way to reset state that allows a processor to > call out to whatever external store it impacts -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2196) Allow the flow to be seen even if there is no ZooKeeper quorum
[ https://issues.apache.org/jira/browse/NIFI-2196?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2196: -- Affects Version/s: 1.0.0 > Allow the flow to be seen even if there is no ZooKeeper quorum > -- > > Key: NIFI-2196 > URL: https://issues.apache.org/jira/browse/NIFI-2196 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > With 1.0.0, in order to view the flow, we have to replicate the request to > get the flow across the cluster so that we can verify permissions, etc. If > unable to connect to ZooKeeper (because of a network issue or a lack of ZK > quorum), we cannot show the UI. > We can improve this by showing the local version of the flow in these cases. > This won't show things like statistics, cluster info, etc. but can at least > show the structure of the flow, based on the permissions that the node > servicing the request believes are accurate. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2195) Improve sophistication of how nodes detect and handle differences between local and cluster flows
[ https://issues.apache.org/jira/browse/NIFI-2195?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2195: -- Fix Version/s: (was: 1.1.0) > Improve sophistication of how nodes detect and handle differences between > local and cluster flows > - > > Key: NIFI-2195 > URL: https://issues.apache.org/jira/browse/NIFI-2195 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > Currently, when a node joins a cluster, it uses the flow's "Fingerprint" to > determine whether or not its flow matches the cluster flow. If not, it will > not join the cluster. There are two main drivers for this. First, we want to > ensure that we don't overwrite the flow with a completely different flow > (though we want to allow for minor differences between the flow, such as the > location of a processor on the graph). Secondly, we want to ensure that there > is no data loss. > We can improve the sophistication of this and allow a node to join the > cluster if there are no 'substantive' differences. It is critical that a node > not inherit a flow that is missing some Connection, if that Connection has > data queued up, as doing so would result in data loss. > We should be smarter, though, about which things really matter. For instance, > if the name of a Processor is different, the node should simply inherit the > name of the Processor from the cluster, instead of refusing to join. > If any changes to the flow are made when joining a cluster, though, we should > ensure that we back up the flow before changing it. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2078) State management for processors whose states are managed externally
[ https://issues.apache.org/jira/browse/NIFI-2078?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2078: -- Affects Version/s: 1.0.0 > State management for processors whose states are managed externally > --- > > Key: NIFI-2078 > URL: https://issues.apache.org/jira/browse/NIFI-2078 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > Inherently by the nature of a given processor it may involve state managed by > itself (using nifi state management), or can be managed by some external > service it interacts with (kafka's offset), and theoretically some might have > both going on. With the new state management, we're giving users a way to > reset state managed by nifi for a given processor. But it doesnt apply to > those processors who have external state. > we should consider offering a way to reset state that allows a processor to > call out to whatever external store it impacts -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2196) Allow the flow to be seen even if there is no ZooKeeper quorum
[ https://issues.apache.org/jira/browse/NIFI-2196?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2196: -- Fix Version/s: (was: 1.1.0) > Allow the flow to be seen even if there is no ZooKeeper quorum > -- > > Key: NIFI-2196 > URL: https://issues.apache.org/jira/browse/NIFI-2196 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > With 1.0.0, in order to view the flow, we have to replicate the request to > get the flow across the cluster so that we can verify permissions, etc. If > unable to connect to ZooKeeper (because of a network issue or a lack of ZK > quorum), we cannot show the UI. > We can improve this by showing the local version of the flow in these cases. > This won't show things like statistics, cluster info, etc. but can at least > show the structure of the flow, based on the permissions that the node > servicing the request believes are accurate. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2293) Add controls for accessing each Node's History
[ https://issues.apache.org/jira/browse/NIFI-2293?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2293: -- Fix Version/s: (was: 1.1.0) > Add controls for accessing each Node's History > -- > > Key: NIFI-2293 > URL: https://issues.apache.org/jira/browse/NIFI-2293 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Matt Gilman > > Each node in a cluster tracks and maintains it's own history. A given node's > events should be accessible from any node in the cluster. As part of > NIFI-2263 the user must go to each node directly in the browser to access > those events. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2325) Add support for LDAPS in authentication provider
[ https://issues.apache.org/jira/browse/NIFI-2325?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2325: -- Affects Version/s: 1.0.0 > Add support for LDAPS in authentication provider > > > Key: NIFI-2325 > URL: https://issues.apache.org/jira/browse/NIFI-2325 > Project: Apache NiFi > Issue Type: Improvement >Affects Versions: 1.0.0 >Reporter: Joseph Witt > > [~mcgilman] [~alopresto] please add thoughts if you have them. > I propose we add support for LDAPS despite StartTLS being the now preferred > approach. This offers more flexibility for use with many of the long > standing LDAP environments out there. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2202) Read Only User/Group Management
[ https://issues.apache.org/jira/browse/NIFI-2202?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2202: -- Affects Version/s: 1.0.0 > Read Only User/Group Management > --- > > Key: NIFI-2202 > URL: https://issues.apache.org/jira/browse/NIFI-2202 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Matt Gilman > > Update the User/Group management page to support users with read only > permissions. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2325) Add support for LDAPS in authentication provider
[ https://issues.apache.org/jira/browse/NIFI-2325?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2325: -- Fix Version/s: (was: 1.1.0) > Add support for LDAPS in authentication provider > > > Key: NIFI-2325 > URL: https://issues.apache.org/jira/browse/NIFI-2325 > Project: Apache NiFi > Issue Type: Improvement >Affects Versions: 1.0.0 >Reporter: Joseph Witt > > [~mcgilman] [~alopresto] please add thoughts if you have them. > I propose we add support for LDAPS despite StartTLS being the now preferred > approach. This offers more flexibility for use with many of the long > standing LDAP environments out there. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2293) Add controls for accessing each Node's History
[ https://issues.apache.org/jira/browse/NIFI-2293?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2293: -- Affects Version/s: 1.0.0 > Add controls for accessing each Node's History > -- > > Key: NIFI-2293 > URL: https://issues.apache.org/jira/browse/NIFI-2293 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Matt Gilman > > Each node in a cluster tracks and maintains it's own history. A given node's > events should be accessible from any node in the cluster. As part of > NIFI-2263 the user must go to each node directly in the browser to access > those events. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2202) Read Only User/Group Management
[ https://issues.apache.org/jira/browse/NIFI-2202?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2202: -- Fix Version/s: (was: 1.1.0) > Read Only User/Group Management > --- > > Key: NIFI-2202 > URL: https://issues.apache.org/jira/browse/NIFI-2202 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Matt Gilman > > Update the User/Group management page to support users with read only > permissions. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2375) Bump Standard processors tika dependency version
[ https://issues.apache.org/jira/browse/NIFI-2375?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2375: -- Fix Version/s: (was: 1.1.0) > Bump Standard processors tika dependency version > - > > Key: NIFI-2375 > URL: https://issues.apache.org/jira/browse/NIFI-2375 > Project: Apache NiFi > Issue Type: Improvement >Affects Versions: 1.0.0 >Reporter: Andre >Assignee: Andre > -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2363) CLUSTER scope state should be viewed/cleared from primary node
[ https://issues.apache.org/jira/browse/NIFI-2363?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2363: -- Fix Version/s: (was: 1.1.0) > CLUSTER scope state should be viewed/cleared from primary node > -- > > Key: NIFI-2363 > URL: https://issues.apache.org/jira/browse/NIFI-2363 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > Currently, every node in a NiFi cluster retrieves or clears CLUSTER scope > state when user view/clear component state from NiFi UI. > Since the cluster-wide state is shared the same value among nodes, those > operation should be executed only from a primary node. > See this discussion for detail: > https://github.com/apache/nifi/pull/563#issuecomment-231393086 -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2364) Node shouldn't be disconnected from a cluster when clear component state operation gets error related to underlying data source
[ https://issues.apache.org/jira/browse/NIFI-2364?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2364: -- Fix Version/s: (was: 1.1.0) > Node shouldn't be disconnected from a cluster when clear component state > operation gets error related to underlying data source > --- > > Key: NIFI-2364 > URL: https://issues.apache.org/jira/browse/NIFI-2364 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > NodeClusterCoordinator.afterRequest() disconnects a node if it couldn't > handle mutation requests such as POST, assuming that node is not functioning > properly. > However, if the cause of issue is related to an external data source, such as > Zookeeper or Kafka, other node will have the same problem even if other node > is elected as a primary node. In such case, disconnecting node wouldn't be a > good recovery solution. Instead, the node should keep connected, and let > users retry the operation. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2364) Node shouldn't be disconnected from a cluster when clear component state operation gets error related to underlying data source
[ https://issues.apache.org/jira/browse/NIFI-2364?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2364: -- Affects Version/s: 1.0.0 > Node shouldn't be disconnected from a cluster when clear component state > operation gets error related to underlying data source > --- > > Key: NIFI-2364 > URL: https://issues.apache.org/jira/browse/NIFI-2364 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > NodeClusterCoordinator.afterRequest() disconnects a node if it couldn't > handle mutation requests such as POST, assuming that node is not functioning > properly. > However, if the cause of issue is related to an external data source, such as > Zookeeper or Kafka, other node will have the same problem even if other node > is elected as a primary node. In such case, disconnecting node wouldn't be a > good recovery solution. Instead, the node should keep connected, and let > users retry the operation. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2375) Bump Standard processors tika dependency version
[ https://issues.apache.org/jira/browse/NIFI-2375?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2375: -- Affects Version/s: 1.0.0 > Bump Standard processors tika dependency version > - > > Key: NIFI-2375 > URL: https://issues.apache.org/jira/browse/NIFI-2375 > Project: Apache NiFi > Issue Type: Improvement >Affects Versions: 1.0.0 >Reporter: Andre >Assignee: Andre > -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2408) If a node is disconnected, it should still provide user ability to see the 'cluster screen'
[ https://issues.apache.org/jira/browse/NIFI-2408?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2408: -- Fix Version/s: (was: 1.1.0) > If a node is disconnected, it should still provide user ability to see the > 'cluster screen' > --- > > Key: NIFI-2408 > URL: https://issues.apache.org/jira/browse/NIFI-2408 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Mark Payne >Assignee: Matt Gilman > > A decision was made a while back that if a node is disconnected from the > cluster, the 'cluster screen' should not be available in the menu in the > top-right corner. However, after using the app like this, I now think it was > a bad decision. The node is still part of the cluster, even though it is > disconnected. Unless the user clicks 'remove', the node is a part of the > cluster, so should be able to render the cluster information. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2432) Clarify message provided when local flow differs from cluster flow
[ https://issues.apache.org/jira/browse/NIFI-2432?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2432: -- Fix Version/s: (was: 1.1.0) > Clarify message provided when local flow differs from cluster flow > -- > > Key: NIFI-2432 > URL: https://issues.apache.org/jira/browse/NIFI-2432 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > Currently, when a node attempts to join a cluster, if its flow is different > than the cluster flow, the message provided is not ideal. For example: > Local Fingerprint:   api.dto.ProcessorDTOPutFileWARN0 secTIMER_DRIVEN1 sec130 > secnullConflict Resolution Strategy=replaceCreate Missing > Directories=trueDirectory=/home/filePermissions=666NO > Cluster Fingerprint: api.dto.ProcessorDTOPutFileWARN0 secTIMER_DRIVEN1 sec130 > secnullConflict Resolution > Strategy=replaceDirectory=/home/filePermissions=666NO_AUTO_TERMINATED_RELATIONSHIPS6 > From this, one can glean the difference between the flow if they understand > the structure of the message, but it is not at all intuitive. At the point > when this message is generated, we have both the proposed flow and the > current flow. We also have methods that will convert both flows into DTO > objects. So we should instead perform a more intelligent 'diff' and output > message such as: > Proposed flow does not contain Process Group ABC(id=1234) > Proposed flow has different property values for Processor MyPutFile(id=4321): > Create Missing Directories = true (Local Flow) > Create Missing Directories = (Cluster Flow) > This makes it much more clear what the difference is between the flows so > that the admin is able to determine how to address the issue. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2437) Enforce HSTS to require HTTPS connections if available
[ https://issues.apache.org/jira/browse/NIFI-2437?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2437: -- Fix Version/s: (was: 1.1.0) > Enforce HSTS to require HTTPS connections if available > -- > > Key: NIFI-2437 > URL: https://issues.apache.org/jira/browse/NIFI-2437 > Project: Apache NiFi > Issue Type: New Feature > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Andy LoPresto > Labels: security > > HTTP Strict Transport Security (HSTS) [1] [2] is a feature of HTTP which > instructs browsers/clients to only communicate with a resource over HTTPS. It > is implemented via a header sent in the response and future connections will > require HTTPS. > [1] https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security > [2] https://www.owasp.org/index.php/HTTP_Strict_Transport_Security_Cheat_Sheet -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2408) If a node is disconnected, it should still provide user ability to see the 'cluster screen'
[ https://issues.apache.org/jira/browse/NIFI-2408?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2408: -- Affects Version/s: 1.0.0 > If a node is disconnected, it should still provide user ability to see the > 'cluster screen' > --- > > Key: NIFI-2408 > URL: https://issues.apache.org/jira/browse/NIFI-2408 > Project: Apache NiFi > Issue Type: Improvement > Components: Core UI >Affects Versions: 1.0.0 >Reporter: Mark Payne >Assignee: Matt Gilman > > A decision was made a while back that if a node is disconnected from the > cluster, the 'cluster screen' should not be available in the menu in the > top-right corner. However, after using the app like this, I now think it was > a bad decision. The node is still part of the cluster, even though it is > disconnected. Unless the user clicks 'remove', the node is a part of the > cluster, so should be able to render the cluster information. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2432) Clarify message provided when local flow differs from cluster flow
[ https://issues.apache.org/jira/browse/NIFI-2432?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2432: -- Affects Version/s: 1.0.0 > Clarify message provided when local flow differs from cluster flow > -- > > Key: NIFI-2432 > URL: https://issues.apache.org/jira/browse/NIFI-2432 > Project: Apache NiFi > Issue Type: Improvement > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Mark Payne > > Currently, when a node attempts to join a cluster, if its flow is different > than the cluster flow, the message provided is not ideal. For example: > Local Fingerprint:   api.dto.ProcessorDTOPutFileWARN0 secTIMER_DRIVEN1 sec130 > secnullConflict Resolution Strategy=replaceCreate Missing > Directories=trueDirectory=/home/filePermissions=666NO > Cluster Fingerprint: api.dto.ProcessorDTOPutFileWARN0 secTIMER_DRIVEN1 sec130 > secnullConflict Resolution > Strategy=replaceDirectory=/home/filePermissions=666NO_AUTO_TERMINATED_RELATIONSHIPS6 > From this, one can glean the difference between the flow if they understand > the structure of the message, but it is not at all intuitive. At the point > when this message is generated, we have both the proposed flow and the > current flow. We also have methods that will convert both flows into DTO > objects. So we should instead perform a more intelligent 'diff' and output > message such as: > Proposed flow does not contain Process Group ABC(id=1234) > Proposed flow has different property values for Processor MyPutFile(id=4321): > Create Missing Directories = true (Local Flow) > Create Missing Directories = (Cluster Flow) > This makes it much more clear what the difference is between the flows so > that the admin is able to determine how to address the issue. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2437) Enforce HSTS to require HTTPS connections if available
[ https://issues.apache.org/jira/browse/NIFI-2437?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2437: -- Affects Version/s: 1.0.0 > Enforce HSTS to require HTTPS connections if available > -- > > Key: NIFI-2437 > URL: https://issues.apache.org/jira/browse/NIFI-2437 > Project: Apache NiFi > Issue Type: New Feature > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Andy LoPresto > Labels: security > > HTTP Strict Transport Security (HSTS) [1] [2] is a feature of HTTP which > instructs browsers/clients to only communicate with a resource over HTTPS. It > is implemented via a header sent in the response and future connections will > require HTTPS. > [1] https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security > [2] https://www.owasp.org/index.php/HTTP_Strict_Transport_Security_Cheat_Sheet -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2442) ExternalStateManager should support Expression Language evaluation
[ https://issues.apache.org/jira/browse/NIFI-2442?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2442: -- Fix Version/s: (was: 1.1.0) > ExternalStateManager should support Expression Language evaluation > -- > > Key: NIFI-2442 > URL: https://issues.apache.org/jira/browse/NIFI-2442 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > Some components have to eagerly get property value change, and do some stuff. > Sometimes, the target properties support Expression Language (EL). > However, since current onPropertyModified method only passes String > representation of property value, we can't evaluate EL in that method > directly. > It would be useful if we can use subject-less EL functions (i.e without > incoming FlowFile) at onPropertyModified, especially with Variable Registry. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2442) ExternalStateManager should support Expression Language evaluation
[ https://issues.apache.org/jira/browse/NIFI-2442?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2442: -- Affects Version/s: 1.0.0 > ExternalStateManager should support Expression Language evaluation > -- > > Key: NIFI-2442 > URL: https://issues.apache.org/jira/browse/NIFI-2442 > Project: Apache NiFi > Issue Type: Sub-task > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Koji Kawamura >Assignee: Koji Kawamura > > Some components have to eagerly get property value change, and do some stuff. > Sometimes, the target properties support Expression Language (EL). > However, since current onPropertyModified method only passes String > representation of property value, we can't evaluate EL in that method > directly. > It would be useful if we can use subject-less EL functions (i.e without > incoming FlowFile) at onPropertyModified, especially with Variable Registry. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2487) Provenance Repository occasionally reporting that a prov file is "Not in GZIP Format"
[
https://issues.apache.org/jira/browse/NIFI-2487?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joseph Witt updated NIFI-2487:
--
Fix Version/s: (was: 1.1.0)
> Provenance Repository occasionally reporting that a prov file is "Not in GZIP
> Format"
> -
>
> Key: NIFI-2487
> URL: https://issues.apache.org/jira/browse/NIFI-2487
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0
>Reporter: Mark Payne
>Assignee: Mark Payne
>
> I occasionally see the following stack trace in my logs:
> {code}
> 2016-08-04 16:24:06,653 ERROR [Timer-Driven Process Thread-11]
> o.a.n.p.PersistentProvenanceRepository Failed to read Provenance Event File
> /nifi/0/repos/prov/1/10585433663.prov.gz due to java.util.zip.ZipException:
> Not in GZIP format
> 2016-08-04 16:24:06,668 ERROR [Timer-Driven Process Thread-11]
> o.a.n.p.PersistentProvenanceRepository
> java.util.zip.ZipException: Not in GZIP format
> at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165)
> ~[na:1.8.0_91]
> at java.util.zip.GZIPInputStream.(GZIPInputStream.java:79)
> ~[na:1.8.0_91]
> at java.util.zip.GZIPInputStream.(GZIPInputStream.java:91)
> ~[na:1.8.0_91]
> at
> org.apache.nifi.provenance.StandardRecordReader.resetStreamForNextBlock(StandardRecordReader.java:151)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.StandardRecordReader.isData(StandardRecordReader.java:438)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.StandardRecordReader.nextRecord(StandardRecordReader.java:273)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.PersistentProvenanceRepository.getEvents(PersistentProvenanceRepository.java:476)
> [nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.PersistentProvenanceRepository.getEvents(PersistentProvenanceRepository.java:448)
> [nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.controller.FlowController.getProvenanceEvents(FlowController.java:3984)
> [nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask.onTrigger(SiteToSiteProvenanceReportingTask.java:235)
> [nifi-site-to-site-reporting-task-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.controller.tasks.ReportingTaskWrapper.run(ReportingTaskWrapper.java:41)
> [nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_91]
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> [na:1.8.0_91]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> [na:1.8.0_91]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> [na:1.8.0_91]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [na:1.8.0_91]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [na:1.8.0_91]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91]
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2487) Provenance Repository occasionally reporting that a prov file is "Not in GZIP Format"
[
https://issues.apache.org/jira/browse/NIFI-2487?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joseph Witt updated NIFI-2487:
--
Affects Version/s: 1.0.0
> Provenance Repository occasionally reporting that a prov file is "Not in GZIP
> Format"
> -
>
> Key: NIFI-2487
> URL: https://issues.apache.org/jira/browse/NIFI-2487
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0
>Reporter: Mark Payne
>Assignee: Mark Payne
>
> I occasionally see the following stack trace in my logs:
> {code}
> 2016-08-04 16:24:06,653 ERROR [Timer-Driven Process Thread-11]
> o.a.n.p.PersistentProvenanceRepository Failed to read Provenance Event File
> /nifi/0/repos/prov/1/10585433663.prov.gz due to java.util.zip.ZipException:
> Not in GZIP format
> 2016-08-04 16:24:06,668 ERROR [Timer-Driven Process Thread-11]
> o.a.n.p.PersistentProvenanceRepository
> java.util.zip.ZipException: Not in GZIP format
> at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165)
> ~[na:1.8.0_91]
> at java.util.zip.GZIPInputStream.(GZIPInputStream.java:79)
> ~[na:1.8.0_91]
> at java.util.zip.GZIPInputStream.(GZIPInputStream.java:91)
> ~[na:1.8.0_91]
> at
> org.apache.nifi.provenance.StandardRecordReader.resetStreamForNextBlock(StandardRecordReader.java:151)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.StandardRecordReader.isData(StandardRecordReader.java:438)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.StandardRecordReader.nextRecord(StandardRecordReader.java:273)
> ~[nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.PersistentProvenanceRepository.getEvents(PersistentProvenanceRepository.java:476)
> [nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.provenance.PersistentProvenanceRepository.getEvents(PersistentProvenanceRepository.java:448)
> [nifi-persistent-provenance-repository-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.controller.FlowController.getProvenanceEvents(FlowController.java:3984)
> [nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask.onTrigger(SiteToSiteProvenanceReportingTask.java:235)
> [nifi-site-to-site-reporting-task-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> org.apache.nifi.controller.tasks.ReportingTaskWrapper.run(ReportingTaskWrapper.java:41)
> [nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_91]
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> [na:1.8.0_91]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> [na:1.8.0_91]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> [na:1.8.0_91]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [na:1.8.0_91]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [na:1.8.0_91]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91]
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[ https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2565: -- Status: Patch Available (was: Open) flagging that there is a patch in play so we don't lose track. either should be in 1.1 or slide to next release but needs discussion to understand status > NiFi processor to parse logs using Grok patterns > > > Key: NIFI-2565 > URL: https://issues.apache.org/jira/browse/NIFI-2565 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Andre > Fix For: 1.1.0 > > > Following up on Ryan Ward to create a Grok capable parser > https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2592) Processor still runs @OnScheduled method when it's running on a non-primary node
[ https://issues.apache.org/jira/browse/NIFI-2592?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2592: -- Fix Version/s: (was: 1.1.0) > Processor still runs @OnScheduled method when it's running on a non-primary > node > > > Key: NIFI-2592 > URL: https://issues.apache.org/jira/browse/NIFI-2592 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Affects Versions: 1.0.0 >Reporter: Jeff Storck > > If a processor is scheduled to run on the primary node only, the instances of > the processor running on non-primary nodes still have their methods annotated > with @OnScheduled invoked. The framework should not invoke the @OnScheduled > methods if the processor isn't supposed to run on that node. > To reproduce this, set up a cluster with the nodes running on the same > server. The ListenHTTP can be scheduled to be run on the primary node only, > but other instances of the processor on non-primary nodes will still try to > bind the port, resulting in exceptions. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2594) UDT Protocol Support, ListenUDT and PutUDT processors
[ https://issues.apache.org/jira/browse/NIFI-2594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2594: -- Fix Version/s: (was: 1.1.0) > UDT Protocol Support, ListenUDT and PutUDT processors > - > > Key: NIFI-2594 > URL: https://issues.apache.org/jira/browse/NIFI-2594 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions > Environment: Netwotk Protocol >Reporter: n h > Labels: Network, Processor, UDT > > Add support for UDT protocol (UDP-based Data Transfer Protocol) : ListenUDT, > PutUDT > https://en.wikipedia.org/wiki/UDP-based_Data_Transfer_Protocol > https://sourceforge.net/projects/udt > https://github.com/dorkbox/UDT > UDP is not reliable. TCP is very slow (in high-speed networks) and > complicated for most scenarios. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2594) UDT Protocol Support, ListenUDT and PutUDT processors
[ https://issues.apache.org/jira/browse/NIFI-2594?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571873#comment-15571873 ] Joseph Witt commented on NIFI-2594: --- removed fix version. Once there is a PR we can align it to a release. Also, this is an awesome idea. > UDT Protocol Support, ListenUDT and PutUDT processors > - > > Key: NIFI-2594 > URL: https://issues.apache.org/jira/browse/NIFI-2594 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions > Environment: Netwotk Protocol >Reporter: n h > Labels: Network, Processor, UDT > > Add support for UDT protocol (UDP-based Data Transfer Protocol) : ListenUDT, > PutUDT > https://en.wikipedia.org/wiki/UDP-based_Data_Transfer_Protocol > https://sourceforge.net/projects/udt > https://github.com/dorkbox/UDT > UDP is not reliable. TCP is very slow (in high-speed networks) and > complicated for most scenarios. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2613) Support extracting content from Microsoft Excel (.xlxs) documents
[ https://issues.apache.org/jira/browse/NIFI-2613?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2613: -- Status: Patch Available (was: Open) flagging as patch avail. needs review > Support extracting content from Microsoft Excel (.xlxs) documents > - > > Key: NIFI-2613 > URL: https://issues.apache.org/jira/browse/NIFI-2613 > Project: Apache NiFi > Issue Type: New Feature > Components: Extensions >Reporter: Jeremy Dyer >Assignee: Jeremy Dyer > Fix For: 1.1.0 > > > Microsoft Excel is a wildly popular application that businesses rely heavily > on to store, visualize, and calculate data. Any single company most likely > has thousands of Excel documents containing data that could be very valuable > if ingested via NiFi and combined with other datasources. Apache POI is a > popular 100% Java library for parsing several Microsoft document formats > including Excel. Apache POI is extremely flexible and can do several things. > This issue would focus solely on using Apache POI to parse an incoming .xlxs > document and convert it to CSV. The processor should be capable of limiting > which excel sheets. CSV seems like the natural choice for outputting each row > since this feature is already available in Excel and feels very natural to > most Excel sheet designs. > This capability should most likely introduce a new "poi" module as I envision > many more capabilities around parsing Microsoft documents could come from > this base effort. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (NIFI-2652) Handle multiple invocations of the encrypt-config tool
[
https://issues.apache.org/jira/browse/NIFI-2652?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joseph Witt updated NIFI-2652:
--
Fix Version/s: (was: 1.1.0)
> Handle multiple invocations of the encrypt-config tool
> --
>
> Key: NIFI-2652
> URL: https://issues.apache.org/jira/browse/NIFI-2652
> Project: Apache NiFi
> Issue Type: Improvement
> Components: Configuration, Tools and Build
>Affects Versions: 1.0.0
>Reporter: Andy LoPresto
>Assignee: Andy LoPresto
> Labels: bootstrap, config, encryption, security
>
> A discussion between [~jtstorck] and myself led to some possible scenarios
> with the {{encrypt-config}} tool. If a user invokes the tool multiple times
> on the same input files (updating in place), what should happen?
> Currently:
> The tool will not operate on any already-protected properties. So sensitive
> properties present and unprotected would be protected by the first
> invocation. If, before the second invocation, new sensitive values were
> provided in the {{nifi.properties}} file, they would be protected by the
> second invocation. If the user provides the same key/password as the first
> invocation, all properties would be encrypted with the same key. However, if
> a different key/password were used, the properties encrypted on the second
> invocation would be protected with a different key, and the new key would
> overwrite the key in the {{bootstrap.conf}} file, rendering the first set of
> properties unrecoverable.
> Possible solutions:
> On invocation of the tool, it first tries to read the existing key from
> {{bootstrap.conf}}. If no key is present, operate as normal.
> * Possibly require second entry of the key/password for confirmation to
> ensure it was not mistyped
> If a key *is* present (one of the following):
> * (Derive if necessary and) validate the key against the existing key and if
> it does not match, fail to run
> * Decrypt any existing encrypted properties with the persisted key and
> re-encrypt all sensitive properties with the new key
> The second option does not require the same key/password to be used
> repeatedly and also provides a mechanism for key migration/rollover.
> Another possible error scenario is if the first invocation was run with the
> JCE unlimited strength cryptographic jurisdiction policies not present (so
> the system was limited to 128-bit encryption) and subsequent invocations are
> run with the policies installed (in which case the tool will attempt to use
> 256-bit encryption). The individual properties will be marked with key
> strength available at the time they were encrypted, so this would also
> indicate the second option above is more robust. However, if the opposite
> flow occurs (256-bit available at first invocation, 128-bit subsequently),
> the tool will not be able to decrypt and re-encrypt the existing protected
> properties.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2661) Create enrichment processor supporting GeoLite ASN
[ https://issues.apache.org/jira/browse/NIFI-2661?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Joseph Witt updated NIFI-2661: -- Fix Version/s: (was: 1.1.0) > Create enrichment processor supporting GeoLite ASN > -- > > Key: NIFI-2661 > URL: https://issues.apache.org/jira/browse/NIFI-2661 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Andre >Assignee: Andre > > Current EnrichGeoIP does not support MaxMind's GeoLite ASN API and database. > It would be great to have a processor capable of doing so. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Assigned] (NIFI-2894) PutSQL property Support Fragmented Transactions documentation fix
[ https://issues.apache.org/jira/browse/NIFI-2894?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Matt Burgess reassigned NIFI-2894: -- Assignee: Matt Burgess > PutSQL property Support Fragmented Transactions documentation fix > - > > Key: NIFI-2894 > URL: https://issues.apache.org/jira/browse/NIFI-2894 > Project: Apache NiFi > Issue Type: Task > Components: Extensions >Affects Versions: 1.0.0 >Reporter: Jeff Storck >Assignee: Matt Burgess >Priority: Minor > Labels: documentation > Fix For: 1.1.0 > > > PutSQL property "Support Fragmented Transactions" has an error in its > documentation, with the proposed edit: > bq. If the fragment.count value is greater than 1, the Processor will not > process any FlowFile -will-+with+ that fragment.identifier until all are > available; -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2894) PutSQL property Support Fragmented Transactions documentation fix
[ https://issues.apache.org/jira/browse/NIFI-2894?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571907#comment-15571907 ] ASF GitHub Bot commented on NIFI-2894: -- GitHub user mattyb149 opened a pull request: https://github.com/apache/nifi/pull/1129 NIFI-2894: Fixed typo in PutSQL documentation You can merge this pull request into a Git repository by running: $ git pull https://github.com/mattyb149/nifi NIFI-2894 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/nifi/pull/1129.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #1129 commit 45e659670a19ab7e47d6345ad3ef0e97fc7e8b43 Author: Matt Burgess Date: 2016-10-13T13:19:08Z NIFI-2894: Fixed typo in PutSQL documentation > PutSQL property Support Fragmented Transactions documentation fix > - > > Key: NIFI-2894 > URL: https://issues.apache.org/jira/browse/NIFI-2894 > Project: Apache NiFi > Issue Type: Task > Components: Extensions >Affects Versions: 1.0.0 >Reporter: Jeff Storck >Assignee: Matt Burgess >Priority: Minor > Labels: documentation > Fix For: 1.1.0 > > > PutSQL property "Support Fragmented Transactions" has an error in its > documentation, with the proposed edit: > bq. If the fragment.count value is greater than 1, the Processor will not > process any FlowFile -will-+with+ that fragment.identifier until all are > available; -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi pull request #1129: NIFI-2894: Fixed typo in PutSQL documentation
GitHub user mattyb149 opened a pull request: https://github.com/apache/nifi/pull/1129 NIFI-2894: Fixed typo in PutSQL documentation You can merge this pull request into a Git repository by running: $ git pull https://github.com/mattyb149/nifi NIFI-2894 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/nifi/pull/1129.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #1129 commit 45e659670a19ab7e47d6345ad3ef0e97fc7e8b43 Author: Matt Burgess Date: 2016-10-13T13:19:08Z NIFI-2894: Fixed typo in PutSQL documentation --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Updated] (NIFI-2894) PutSQL property Support Fragmented Transactions documentation fix
[ https://issues.apache.org/jira/browse/NIFI-2894?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Matt Burgess updated NIFI-2894: --- Status: Patch Available (was: In Progress) > PutSQL property Support Fragmented Transactions documentation fix > - > > Key: NIFI-2894 > URL: https://issues.apache.org/jira/browse/NIFI-2894 > Project: Apache NiFi > Issue Type: Task > Components: Extensions >Affects Versions: 1.0.0 >Reporter: Jeff Storck >Assignee: Matt Burgess >Priority: Minor > Labels: documentation > Fix For: 1.1.0 > > > PutSQL property "Support Fragmented Transactions" has an error in its > documentation, with the proposed edit: > bq. If the fragment.count value is greater than 1, the Processor will not > process any FlowFile -will-+with+ that fragment.identifier until all are > available; -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2737) Max Value Column in QueryDatabaseTable/GenerateTableFetch need to handle rollover
[ https://issues.apache.org/jira/browse/NIFI-2737?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571932#comment-15571932 ] Matt Burgess commented on NIFI-2737: What kind of column types and rollover do you envision? For a time dimension, the entries for things like day of month, month, and year are likely integers, but the processor would need to know that month rolls over at value 12, etc. For pure types like Integer, even with variable length, we could calculate the maximum possible value, but that's the only use case I can think of without adding many more properties (likely dynamic) for the user to indicate rollover values for max-value columns. Not saying it's a bad idea, just want to hear your thoughts on the approach :) > Max Value Column in QueryDatabaseTable/GenerateTableFetch need to handle > rollover > - > > Key: NIFI-2737 > URL: https://issues.apache.org/jira/browse/NIFI-2737 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Reporter: Jeff Storck >Priority: Minor > Fix For: 1.1.0 > > > QueryDatabaseTable and GenerateTableFetch should support the case where a max > value column rolls over. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-1991) Zero Master Cluster: Modifying a disconnected node causes other nodes to not be able to join cluster
[ https://issues.apache.org/jira/browse/NIFI-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571948#comment-15571948 ] Andrew Lim commented on NIFI-1991: -- [~markap14], thanks for the comments. I agree that what has already been put in place for cluster management is sufficient. > Zero Master Cluster: Modifying a disconnected node causes other nodes to not > be able to join cluster > - > > Key: NIFI-1991 > URL: https://issues.apache.org/jira/browse/NIFI-1991 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Andrew Lim > > I created a zero master cluster with 3 nodes. All three were up and running. > I stopped all 3 instances and then brought up just Node1. > In the Node1 UI, I made some edits to the flow. > I attempted but was not able to bring up Node2 and Node3 to join the cluster. > Here is what I saw in the logs: > 2016-06-09 14:35:47,051 WARN [main] org.apache.nifi.web.server.JettyServer > Failed to start web server... shutting down. > java.lang.Exception: Unable to load flow due to: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:753) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.(NiFi.java:137) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.main(NiFi.java:227) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > Caused by: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:501) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:744) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 2 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Failed to > connect node to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:862) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:497) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 3 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Proposed > configuration is not inheritable by the flow controller because of flow > differences: Found difference in Flows: > Local Fingerprint contains additional configuration from Cluster Fingerprint: > eba77dac-32ad-356b-8a82-f669d046aa21eba77dac-32ad-356b-8a82-f669d046aa21org.apache.nifi.processors.standard.LogAttributeNO_VALUEAttributes > to IgnoreNO_VALUEAttributes to LogNO_VALUELog prefixNO_VALUEs > at > org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:219) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1329) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:75) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:668) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:839) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 4 common frames omitted > 2016-06-09 14:35:47,052 INFO [Thread-1] org.apache.nifi.NiFi Initiating > shutdown of Jetty web server... -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi-minifi-cpp issue #18: MINIFI-34 - attempt to progress the CMake environ...
Github user apiri commented on the issue: https://github.com/apache/nifi-minifi-cpp/pull/18 Yeah, seems like we have a good start. Will do some testing around on a few systems and get it merged in today. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Resolved] (NIFI-1991) Zero Master Cluster: Modifying a disconnected node causes other nodes to not be able to join cluster
[ https://issues.apache.org/jira/browse/NIFI-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Andrew Lim resolved NIFI-1991. -- Resolution: Won't Fix Closing per discussion. > Zero Master Cluster: Modifying a disconnected node causes other nodes to not > be able to join cluster > - > > Key: NIFI-1991 > URL: https://issues.apache.org/jira/browse/NIFI-1991 > Project: Apache NiFi > Issue Type: Bug > Components: Core Framework >Reporter: Andrew Lim > > I created a zero master cluster with 3 nodes. All three were up and running. > I stopped all 3 instances and then brought up just Node1. > In the Node1 UI, I made some edits to the flow. > I attempted but was not able to bring up Node2 and Node3 to join the cluster. > Here is what I saw in the logs: > 2016-06-09 14:35:47,051 WARN [main] org.apache.nifi.web.server.JettyServer > Failed to start web server... shutting down. > java.lang.Exception: Unable to load flow due to: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:753) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.(NiFi.java:137) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.NiFi.main(NiFi.java:227) > [nifi-runtime-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > Caused by: java.io.IOException: > org.apache.nifi.controller.UninheritableFlowException: Failed to connect node > to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:501) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:744) > ~[nifi-jetty-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 2 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Failed to > connect node to cluster because local flow is different than cluster flow. > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:862) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:497) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 3 common frames omitted > Caused by: org.apache.nifi.controller.UninheritableFlowException: Proposed > configuration is not inheritable by the flow controller because of flow > differences: Found difference in Flows: > Local Fingerprint contains additional configuration from Cluster Fingerprint: > eba77dac-32ad-356b-8a82-f669d046aa21eba77dac-32ad-356b-8a82-f669d046aa21org.apache.nifi.processors.standard.LogAttributeNO_VALUEAttributes > to IgnoreNO_VALUEAttributes to LogNO_VALUELog prefixNO_VALUEs > at > org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:219) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1329) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:75) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:668) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > at > org.apache.nifi.controller.StandardFlowService.loadFromConnectionResponse(StandardFlowService.java:839) > ~[nifi-framework-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] > ... 4 common frames omitted > 2016-06-09 14:35:47,052 INFO [Thread-1] org.apache.nifi.NiFi Initiating > shutdown of Jetty web server... -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Assigned] (NIFI-2819) Improve ModifyBytes (Add expression language support)
[ https://issues.apache.org/jira/browse/NIFI-2819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Matt Burgess reassigned NIFI-2819: -- Assignee: Matt Burgess > Improve ModifyBytes (Add expression language support) > - > > Key: NIFI-2819 > URL: https://issues.apache.org/jira/browse/NIFI-2819 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Affects Versions: 1.0.0 >Reporter: n h >Assignee: Matt Burgess > Fix For: 1.1.0 > > > Add expression language support to "Start Offset" and "End Offset" -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi pull request #1122: NIFI-2863: S2S to allow cluster URL more leniently
Github user asfgit closed the pull request at: https://github.com/apache/nifi/pull/1122 --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-2863) A remote process group pointed to a host without the trailing "/nifi" will fail with mis-leading bulletins
[
https://issues.apache.org/jira/browse/NIFI-2863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571982#comment-15571982
]
ASF subversion and git services commented on NIFI-2863:
---
Commit c470fae0653add23ff5ceaf04a814b98f3f612cb in nifi's branch
refs/heads/master from [~ijokarumawak]
[ https://git-wip-us.apache.org/repos/asf?p=nifi.git;h=c470fae ]
NIFI-2863: S2S to allow cluster URL more leniently. This closes #1122
- Consolidated the target cluster URL resolving logic into
SiteToSiteRestApiClient's as a common method
- Changed to more descriptive error message
- Added more unit test cases
> A remote process group pointed to a host without the trailing "/nifi" will
> fail with mis-leading bulletins
> --
>
> Key: NIFI-2863
> URL: https://issues.apache.org/jira/browse/NIFI-2863
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.0.0
>Reporter: Joseph Percivall
>Assignee: Koji Kawamura
> Fix For: 1.1.0
>
>
> To replicate:
> 1: Set up NiFi instance on port 8080 and remote port 8081 (unsecure S2S)
> 2: create input port
> 3: create RPG pointing to "http://localhost:8080";
> This RPG will correctly get the instance name, listing of ports and port
> status but when transmission is enabled and a flowfile is queued to be sent
> the following error is generated:
> "RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]"
> Looking at the logs there is this message:
> 2016-10-04 14:11:34,298 WARN [Timer-Driven Process Thread-2]
> o.a.n.r.util.SiteToSiteRestApiClient Failed to parse Json, response=
>
>
>
>
>
> NiFi
>
> />
> type="text/css" />
> href="/nifi/assets/font-awesome/css/font-awesome.min.css" type="text/css" />
> type="text/css" />
> type="text/css" />
>
>
>
>
> Did you mean: /nifi
>
> You may have mistyped...
>
>
>
> 2016-10-04 14:11:34,298 ERROR [Timer-Driven Process Thread-2]
> o.a.nifi.remote.StandardRemoteGroupPort
> RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]
> This should either be fixed (to allow without "/nifi") or explicitly
> validated.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[GitHub] nifi issue #1122: NIFI-2863: S2S to allow cluster URL more leniently
Github user mcgilman commented on the issue: https://github.com/apache/nifi/pull/1122 Thanks @ijokarumawak! This has been merged to master. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-2863) A remote process group pointed to a host without the trailing "/nifi" will fail with mis-leading bulletins
[
https://issues.apache.org/jira/browse/NIFI-2863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571987#comment-15571987
]
ASF GitHub Bot commented on NIFI-2863:
--
Github user mcgilman commented on the issue:
https://github.com/apache/nifi/pull/1122
Thanks @ijokarumawak! This has been merged to master.
> A remote process group pointed to a host without the trailing "/nifi" will
> fail with mis-leading bulletins
> --
>
> Key: NIFI-2863
> URL: https://issues.apache.org/jira/browse/NIFI-2863
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.0.0
>Reporter: Joseph Percivall
>Assignee: Koji Kawamura
> Fix For: 1.1.0
>
>
> To replicate:
> 1: Set up NiFi instance on port 8080 and remote port 8081 (unsecure S2S)
> 2: create input port
> 3: create RPG pointing to "http://localhost:8080";
> This RPG will correctly get the instance name, listing of ports and port
> status but when transmission is enabled and a flowfile is queued to be sent
> the following error is generated:
> "RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]"
> Looking at the logs there is this message:
> 2016-10-04 14:11:34,298 WARN [Timer-Driven Process Thread-2]
> o.a.n.r.util.SiteToSiteRestApiClient Failed to parse Json, response=
>
>
>
>
>
> NiFi
>
> />
> type="text/css" />
> href="/nifi/assets/font-awesome/css/font-awesome.min.css" type="text/css" />
> type="text/css" />
> type="text/css" />
>
>
>
>
> Did you mean: /nifi
>
> You may have mistyped...
>
>
>
> 2016-10-04 14:11:34,298 ERROR [Timer-Driven Process Thread-2]
> o.a.nifi.remote.StandardRemoteGroupPort
> RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]
> This should either be fixed (to allow without "/nifi") or explicitly
> validated.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2863) A remote process group pointed to a host without the trailing "/nifi" will fail with mis-leading bulletins
[
https://issues.apache.org/jira/browse/NIFI-2863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571984#comment-15571984
]
ASF GitHub Bot commented on NIFI-2863:
--
Github user asfgit closed the pull request at:
https://github.com/apache/nifi/pull/1122
> A remote process group pointed to a host without the trailing "/nifi" will
> fail with mis-leading bulletins
> --
>
> Key: NIFI-2863
> URL: https://issues.apache.org/jira/browse/NIFI-2863
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.0.0
>Reporter: Joseph Percivall
>Assignee: Koji Kawamura
> Fix For: 1.1.0
>
>
> To replicate:
> 1: Set up NiFi instance on port 8080 and remote port 8081 (unsecure S2S)
> 2: create input port
> 3: create RPG pointing to "http://localhost:8080";
> This RPG will correctly get the instance name, listing of ports and port
> status but when transmission is enabled and a flowfile is queued to be sent
> the following error is generated:
> "RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]"
> Looking at the logs there is this message:
> 2016-10-04 14:11:34,298 WARN [Timer-Driven Process Thread-2]
> o.a.n.r.util.SiteToSiteRestApiClient Failed to parse Json, response=
>
>
>
>
>
> NiFi
>
> />
> type="text/css" />
> href="/nifi/assets/font-awesome/css/font-awesome.min.css" type="text/css" />
> type="text/css" />
> type="text/css" />
>
>
>
>
> Did you mean: /nifi
>
> You may have mistyped...
>
>
>
> 2016-10-04 14:11:34,298 ERROR [Timer-Driven Process Thread-2]
> o.a.nifi.remote.StandardRemoteGroupPort
> RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]
> This should either be fixed (to allow without "/nifi") or explicitly
> validated.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[GitHub] nifi pull request #1130: NIFI-2819: Added support for Expresssion Language i...
GitHub user mattyb149 opened a pull request: https://github.com/apache/nifi/pull/1130 NIFI-2819: Added support for Expresssion Language in ModifyBytes You can merge this pull request into a Git repository by running: $ git pull https://github.com/mattyb149/nifi NIFI-2819 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/nifi/pull/1130.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #1130 commit 0c4b9e56e1729723039339ceee4dc78e0cd74862 Author: Matt Burgess Date: 2016-10-13T13:50:56Z NIFI-2819: Added support for Expresssion Language in ModifyBytes --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #1127: NIFI-2861 ControlRate should accept more than one f...
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83220736
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -228,12 +238,13 @@ public void onScheduled(final ProcessContext context)
{
rateControlAttribute =
context.getProperty(RATE_CONTROL_ATTRIBUTE_NAME).getValue();
maximumRateStr =
context.getProperty(MAX_RATE).getValue().toUpperCase();
groupingAttributeName =
context.getProperty(GROUPING_ATTRIBUTE_NAME).getValue();
+maxFlowFilePerTrigger =
context.getProperty(MAX_FF_PER_TRIGGER).getValue();
--- End diff --
This should probably be defined as an int, rather than a String, and can
then just use context.getProperty().asInteger(). But I really prefer to remove
this property all together.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (NIFI-2819) Improve ModifyBytes (Add expression language support)
[ https://issues.apache.org/jira/browse/NIFI-2819?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571991#comment-15571991 ] ASF GitHub Bot commented on NIFI-2819: -- GitHub user mattyb149 opened a pull request: https://github.com/apache/nifi/pull/1130 NIFI-2819: Added support for Expresssion Language in ModifyBytes You can merge this pull request into a Git repository by running: $ git pull https://github.com/mattyb149/nifi NIFI-2819 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/nifi/pull/1130.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #1130 commit 0c4b9e56e1729723039339ceee4dc78e0cd74862 Author: Matt Burgess Date: 2016-10-13T13:50:56Z NIFI-2819: Added support for Expresssion Language in ModifyBytes > Improve ModifyBytes (Add expression language support) > - > > Key: NIFI-2819 > URL: https://issues.apache.org/jira/browse/NIFI-2819 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Affects Versions: 1.0.0 >Reporter: n h >Assignee: Matt Burgess > Fix For: 1.1.0 > > > Add expression language support to "Start Offset" and "End Offset" -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi pull request #1127: NIFI-2861 ControlRate should accept more than one f...
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83218149
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
+
+ThrottleFilter(final String ffPerTrigger) {
+super();
+flowFilesPerTrigger = ffPerTrigger == null ? 1L :
Long.parseLong(ffPerTrigger);
--- End diff --
Should probably be passed in an int or a long, rather than a String
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Updated] (NIFI-2819) Improve ModifyBytes (Add expression language support)
[ https://issues.apache.org/jira/browse/NIFI-2819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Matt Burgess updated NIFI-2819: --- Status: Patch Available (was: In Progress) > Improve ModifyBytes (Add expression language support) > - > > Key: NIFI-2819 > URL: https://issues.apache.org/jira/browse/NIFI-2819 > Project: Apache NiFi > Issue Type: Improvement > Components: Extensions >Affects Versions: 1.0.0 >Reporter: n h >Assignee: Matt Burgess > Fix For: 1.1.0 > > > Add expression language support to "Start Offset" and "End Offset" -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2861) ControlRate should accept more than one flow file per execution
[
https://issues.apache.org/jira/browse/NIFI-2861?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571999#comment-15571999
]
ASF GitHub Bot commented on NIFI-2861:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83218107
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
--- End diff --
This filter is not thread-safe... don't think we need an AtomicLong here.
Can just use an int.
> ControlRate should accept more than one flow file per execution
> ---
>
> Key: NIFI-2861
> URL: https://issues.apache.org/jira/browse/NIFI-2861
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0, 0.7.0
>Reporter: Joe Skora
>Assignee: Joe Skora
>
> The {{ControlRate}} processor implements a {{FlowFileFilter}} that returns
> the {{FlowFileFilter.ACCEPT_AND_TERMINATE}} result if the {{FlowFile}} fits
> with the rate limit, affectively limiting it to one {{FlowFile}} per
> {{ConrolRate.onTrigger()}} invocation. This is a significant bottleneck when
> processing very large quantities of small files making it unlikely to hit the
> rate limits.
> It should allow multiple files, perhaps with a configurable maximum, per
> {{ControlRate.onTrigger()}} invocation by issuing the
> {{FlowFileFilter.ACCEPT_AND_CONTINUE}} result until the limits are reached.
> In a preliminary test this eliminated the bottleneck.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Created] (NIFI-2896) Perform Release Management Functions for 0.7.1
Joe Skora created NIFI-2896: --- Summary: Perform Release Management Functions for 0.7.1 Key: NIFI-2896 URL: https://issues.apache.org/jira/browse/NIFI-2896 Project: Apache NiFi Issue Type: Task Reporter: Joe Skora Assignee: Joe Skora Fix For: 0.7.1 -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi pull request #1127: NIFI-2861 ControlRate should accept more than one f...
Github user markap14 commented on a diff in the pull request: https://github.com/apache/nifi/pull/1127#discussion_r83217567 --- Diff: nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java --- @@ -115,6 +115,13 @@ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(false) .build(); +public static final PropertyDescriptor MAX_FF_PER_TRIGGER = new PropertyDescriptor.Builder() --- End diff -- @jskora I'm not sure that this needs to be configurable. This is an implementation detail that feels a bit leaky to me. Users do not know what an 'onTrigger() call' is. We should probably just cap it at say 1000 and not more than the max number of FlowFiles to transfer per 'Time Duration'. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #1127: NIFI-2861 ControlRate should accept more than one f...
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83220874
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
+
+ThrottleFilter(final String ffPerTrigger) {
+super();
--- End diff --
The parent class here is Object. I don't think there's a need to call
super()
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] nifi pull request #1127: NIFI-2861 ControlRate should accept more than one f...
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83218107
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
--- End diff --
This filter is not thread-safe... don't think we need an AtomicLong here.
Can just use an int.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (NIFI-2861) ControlRate should accept more than one flow file per execution
[
https://issues.apache.org/jira/browse/NIFI-2861?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572000#comment-15572000
]
ASF GitHub Bot commented on NIFI-2861:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83220736
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -228,12 +238,13 @@ public void onScheduled(final ProcessContext context)
{
rateControlAttribute =
context.getProperty(RATE_CONTROL_ATTRIBUTE_NAME).getValue();
maximumRateStr =
context.getProperty(MAX_RATE).getValue().toUpperCase();
groupingAttributeName =
context.getProperty(GROUPING_ATTRIBUTE_NAME).getValue();
+maxFlowFilePerTrigger =
context.getProperty(MAX_FF_PER_TRIGGER).getValue();
--- End diff --
This should probably be defined as an int, rather than a String, and can
then just use context.getProperty().asInteger(). But I really prefer to remove
this property all together.
> ControlRate should accept more than one flow file per execution
> ---
>
> Key: NIFI-2861
> URL: https://issues.apache.org/jira/browse/NIFI-2861
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0, 0.7.0
>Reporter: Joe Skora
>Assignee: Joe Skora
>
> The {{ControlRate}} processor implements a {{FlowFileFilter}} that returns
> the {{FlowFileFilter.ACCEPT_AND_TERMINATE}} result if the {{FlowFile}} fits
> with the rate limit, affectively limiting it to one {{FlowFile}} per
> {{ConrolRate.onTrigger()}} invocation. This is a significant bottleneck when
> processing very large quantities of small files making it unlikely to hit the
> rate limits.
> It should allow multiple files, perhaps with a configurable maximum, per
> {{ControlRate.onTrigger()}} invocation by issuing the
> {{FlowFileFilter.ACCEPT_AND_CONTINUE}} result until the limits are reached.
> In a preliminary test this eliminated the bottleneck.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2861) ControlRate should accept more than one flow file per execution
[
https://issues.apache.org/jira/browse/NIFI-2861?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571998#comment-15571998
]
ASF GitHub Bot commented on NIFI-2861:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83218149
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
+
+ThrottleFilter(final String ffPerTrigger) {
+super();
+flowFilesPerTrigger = ffPerTrigger == null ? 1L :
Long.parseLong(ffPerTrigger);
--- End diff --
Should probably be passed in an int or a long, rather than a String
> ControlRate should accept more than one flow file per execution
> ---
>
> Key: NIFI-2861
> URL: https://issues.apache.org/jira/browse/NIFI-2861
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0, 0.7.0
>Reporter: Joe Skora
>Assignee: Joe Skora
>
> The {{ControlRate}} processor implements a {{FlowFileFilter}} that returns
> the {{FlowFileFilter.ACCEPT_AND_TERMINATE}} result if the {{FlowFile}} fits
> with the rate limit, affectively limiting it to one {{FlowFile}} per
> {{ConrolRate.onTrigger()}} invocation. This is a significant bottleneck when
> processing very large quantities of small files making it unlikely to hit the
> rate limits.
> It should allow multiple files, perhaps with a configurable maximum, per
> {{ControlRate.onTrigger()}} invocation by issuing the
> {{FlowFileFilter.ACCEPT_AND_CONTINUE}} result until the limits are reached.
> In a preliminary test this eliminated the bottleneck.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2861) ControlRate should accept more than one flow file per execution
[
https://issues.apache.org/jira/browse/NIFI-2861?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571996#comment-15571996
]
ASF GitHub Bot commented on NIFI-2861:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83220874
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -381,6 +392,14 @@ public boolean tryAdd(final long value) {
private class ThrottleFilter implements FlowFileFilter {
+private final long flowFilesPerTrigger;
+private final AtomicLong flowFilesFiltered = new AtomicLong(0L);
+
+ThrottleFilter(final String ffPerTrigger) {
+super();
--- End diff --
The parent class here is Object. I don't think there's a need to call
super()
> ControlRate should accept more than one flow file per execution
> ---
>
> Key: NIFI-2861
> URL: https://issues.apache.org/jira/browse/NIFI-2861
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0, 0.7.0
>Reporter: Joe Skora
>Assignee: Joe Skora
>
> The {{ControlRate}} processor implements a {{FlowFileFilter}} that returns
> the {{FlowFileFilter.ACCEPT_AND_TERMINATE}} result if the {{FlowFile}} fits
> with the rate limit, affectively limiting it to one {{FlowFile}} per
> {{ConrolRate.onTrigger()}} invocation. This is a significant bottleneck when
> processing very large quantities of small files making it unlikely to hit the
> rate limits.
> It should allow multiple files, perhaps with a configurable maximum, per
> {{ControlRate.onTrigger()}} invocation by issuing the
> {{FlowFileFilter.ACCEPT_AND_CONTINUE}} result until the limits are reached.
> In a preliminary test this eliminated the bottleneck.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2861) ControlRate should accept more than one flow file per execution
[
https://issues.apache.org/jira/browse/NIFI-2861?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15571997#comment-15571997
]
ASF GitHub Bot commented on NIFI-2861:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1127#discussion_r83217567
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
---
@@ -115,6 +115,13 @@
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(false)
.build();
+public static final PropertyDescriptor MAX_FF_PER_TRIGGER = new
PropertyDescriptor.Builder()
--- End diff --
@jskora I'm not sure that this needs to be configurable. This is an
implementation detail that feels a bit leaky to me. Users do not know what an
'onTrigger() call' is. We should probably just cap it at say 1000 and not more
than the max number of FlowFiles to transfer per 'Time Duration'.
> ControlRate should accept more than one flow file per execution
> ---
>
> Key: NIFI-2861
> URL: https://issues.apache.org/jira/browse/NIFI-2861
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
>Affects Versions: 1.0.0, 0.7.0
>Reporter: Joe Skora
>Assignee: Joe Skora
>
> The {{ControlRate}} processor implements a {{FlowFileFilter}} that returns
> the {{FlowFileFilter.ACCEPT_AND_TERMINATE}} result if the {{FlowFile}} fits
> with the rate limit, affectively limiting it to one {{FlowFile}} per
> {{ConrolRate.onTrigger()}} invocation. This is a significant bottleneck when
> processing very large quantities of small files making it unlikely to hit the
> rate limits.
> It should allow multiple files, perhaps with a configurable maximum, per
> {{ControlRate.onTrigger()}} invocation by issuing the
> {{FlowFileFilter.ACCEPT_AND_CONTINUE}} result until the limits are reached.
> In a preliminary test this eliminated the bottleneck.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Updated] (NIFI-2863) A remote process group pointed to a host without the trailing "/nifi" will fail with mis-leading bulletins
[
https://issues.apache.org/jira/browse/NIFI-2863?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Matt Gilman updated NIFI-2863:
--
Resolution: Fixed
Status: Resolved (was: Patch Available)
> A remote process group pointed to a host without the trailing "/nifi" will
> fail with mis-leading bulletins
> --
>
> Key: NIFI-2863
> URL: https://issues.apache.org/jira/browse/NIFI-2863
> Project: Apache NiFi
> Issue Type: Bug
>Affects Versions: 1.0.0
>Reporter: Joseph Percivall
>Assignee: Koji Kawamura
> Fix For: 1.1.0
>
>
> To replicate:
> 1: Set up NiFi instance on port 8080 and remote port 8081 (unsecure S2S)
> 2: create input port
> 3: create RPG pointing to "http://localhost:8080";
> This RPG will correctly get the instance name, listing of ports and port
> status but when transmission is enabled and a flowfile is queued to be sent
> the following error is generated:
> "RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]"
> Looking at the logs there is this message:
> 2016-10-04 14:11:34,298 WARN [Timer-Driven Process Thread-2]
> o.a.n.r.util.SiteToSiteRestApiClient Failed to parse Json, response=
>
>
>
>
>
> NiFi
>
> />
> type="text/css" />
> href="/nifi/assets/font-awesome/css/font-awesome.min.css" type="text/css" />
> type="text/css" />
> type="text/css" />
>
>
>
>
> Did you mean: /nifi
>
> You may have mistyped...
>
>
>
> 2016-10-04 14:11:34,298 ERROR [Timer-Driven Process Thread-2]
> o.a.nifi.remote.StandardRemoteGroupPort
> RemoteGroupPort[name=test1,target=http://localhost:8080] failed to
> communicate with http://localhost:8080 due to
> org.codehaus.jackson.JsonParseException: Unexpected character ('<' (code
> 60)): expected a valid value (number, String, array, object, 'true', 'false'
> or 'null')
> at [Source: java.io.StringReader@44d519ce; line: 3, column: 2]
> This should either be fixed (to allow without "/nifi") or explicitly
> validated.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2896) Perform Release Management Functions for 0.7.1
[ https://issues.apache.org/jira/browse/NIFI-2896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572029#comment-15572029 ] Joe Skora commented on NIFI-2896: - Per email threads[1] and [2], basing the release on 40618364e70a966f9c1e425674b53b22b1fb0fb0. [1]http://mail-archives.apache.org/mod_mbox/nifi-dev/201609.mbox/%3CCACkT4waF3f7W%3DbeJp%2BbnV-OD5D6XfVkRQpypMao1hBSkg7irzg%40mail.gmail.com%3E [2]http://mail-archives.apache.org/mod_mbox/nifi-dev/201610.mbox/%3CCA%2BLyY57EWffhWfpJ-DYaCtLwW8%3DgJ8kX8%3DQdWtGu5a873WyN4A%40mail.gmail.com%3E > Perform Release Management Functions for 0.7.1 > -- > > Key: NIFI-2896 > URL: https://issues.apache.org/jira/browse/NIFI-2896 > Project: Apache NiFi > Issue Type: Task >Reporter: Joe Skora >Assignee: Joe Skora > Labels: release > Fix For: 0.7.1 > > -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[ https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572044#comment-15572044 ] ASF GitHub Bot commented on NIFI-2565: -- Github user markap14 commented on the issue: https://github.com/apache/nifi/pull/1108 @selim-namsi Thanks for contributing this! I have actually been very interested in using NiFi to do some log parsing but hadn't really dug in very much to understand the best way to go about it. This looks like it could be very powerful! Before we get this merged into the codebase, though, it looks like there is some work that needs to be done to the PR. The concern stems, I think, from you not yet being overly familiar with the API, as there are empty @ReadsAttributes, @WritesAttributes annotations, etc. But the great news is that the NiFi community tends to be very inclusive and will help to get everything in great shape! One thing that I did notice is that you updated the Licensing information, which is one of the most commonly overlooked issues. So very glad that's there. I'll leave some inline feedback on things that I notice, but very much looking forward to this getting in! > NiFi processor to parse logs using Grok patterns > > > Key: NIFI-2565 > URL: https://issues.apache.org/jira/browse/NIFI-2565 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Andre > Fix For: 1.1.0 > > > Following up on Ryan Ward to create a Grok capable parser > https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[GitHub] nifi issue #1108: NIFI-2565: add Grok parser
Github user markap14 commented on the issue: https://github.com/apache/nifi/pull/1108 @selim-namsi Thanks for contributing this! I have actually been very interested in using NiFi to do some log parsing but hadn't really dug in very much to understand the best way to go about it. This looks like it could be very powerful! Before we get this merged into the codebase, though, it looks like there is some work that needs to be done to the PR. The concern stems, I think, from you not yet being overly familiar with the API, as there are empty @ReadsAttributes, @WritesAttributes annotations, etc. But the great news is that the NiFi community tends to be very inclusive and will help to get everything in great shape! One thing that I did notice is that you updated the Licensing information, which is one of the most commonly overlooked issues. So very glad that's there. I'll leave some inline feedback on things that I notice, but very much looking forward to this getting in! --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[jira] [Commented] (NIFI-2893) Missing or wrong API doc
[
https://issues.apache.org/jira/browse/NIFI-2893?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572072#comment-15572072
]
Matt Gilman commented on NIFI-2893:
---
The controller services endpoint is not under /process-groups. The endpoints
are grouped largely according to the resource that's used to authorize the
request. Because controller services are a cross-cutting concern (meaning
different processors with potentially different access policies can reference
the same services) access to them is provided through
- /flow/process-groups/{id}/controller-services
- /flow/controller/controller-services
The /flow endpoints are authorized with the resource that provides access to
the flow (flow structure... what components, including services, are connected
but not including what the components actually are) and the UI.
Also, I looked into the documentation reporting a 200 response for all POSTs
requests. This is a bug in swagger [1][2] that's generating the incorrect
response code. I tried out a couple of the suggestions and none of them worked
as I would like it. I'd like to keep this JIRA open until we're able to pull in
a newer version of swagger that addresses the incorrect response code.
[1] https://github.com/kongchen/swagger-maven-plugin/issues/107
[2] https://github.com/kongchen/swagger-maven-plugin/issues/216
> Missing or wrong API doc
>
>
> Key: NIFI-2893
> URL: https://issues.apache.org/jira/browse/NIFI-2893
> Project: Apache NiFi
> Issue Type: Bug
> Components: Documentation & Website
>Affects Versions: 1.0.0
>Reporter: Stephane Maarek
>Priority: Minor
>
> At:
> https://nifi.apache.org/docs/nifi-docs/rest-api/
> Missing:
> - GET /process-groups/{id}/controller-services
> Erroneous:
> - POST /process-groups/{id}/templates/upload
> > should give a 201 in case of created, returns Location in header so we
> can get the ID out of it
> > it'd be good to have a description of the returned xml (nodes, etc)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572097#comment-15572097
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83229335
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGrokParser.java
---
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Created by snamsi on 05/10/16.
+ */
+public class TestGrokParser {
+
+private TestRunner testRunner;
+final static Path GROK_LOG_INPUT =
Paths.get("src/test/resources/TestGrokParser/apache.log");
+final static Path GROK_TEXT_INPUT =
Paths.get("src/test/resources/TestGrokParser/simple_text.log");
+
+
+@Before
+public void init() {
+testRunner = TestRunners.newTestRunner(GrokParser.class);
+}
+
+@Test
+public void testGrokParserWithMatchedContent() throws IOException {
+
+
+testRunner.setProperty(GrokParser.GROK_EXPRESSION,
"%{COMMONAPACHELOG}");
+testRunner.setProperty(GrokParser.GROK_PATTERN_FILE,
"src/test/resources/TestGrokParser/patterns");
+testRunner.enqueue(GROK_LOG_INPUT);
+testRunner.run();
+testRunner.assertAllFlowFilesTransferred(GrokParser.REL_MATCH);
+final MockFlowFile matched =
testRunner.getFlowFilesForRelationship(GrokParser.REL_MATCH).get(0);
+
+matched.assertAttributeEquals("verb","GET");
+matched.assertAttributeEquals("response","401");
+matched.assertAttributeEquals("bytes","12846");
+matched.assertAttributeEquals("clientip","64.242.88.10");
+matched.assertAttributeEquals("auth","-");
+matched.assertAttributeEquals("timestamp","07/Mar/2004:16:05:49
-0800");
+
matched.assertAttributeEquals("request","/twiki/bin/edit/Main/Double_bounce_sender?topicparent=Main.ConfigurationVariables");
+matched.assertAttributeEquals("httpversion","1.1");
+
+}
+
+@Test
+public void testGrokParserWithUnMatchedContent() throws IOException {
+
+
+testRunner.setProperty(GrokParser.GROK_EXPRESSION, "%{ADDRESS}");
+testRunner.setProperty(GrokParser.GROK_PATTERN_FILE,
"src/test/resources/TestGrokParser/patterns");
+testRunner.enqueue(GROK_TEXT_INPUT);
+testRunner.run();
+testRunner.assertAllFlowFilesTransferred(GrokParser.REL_NO_MATCH);
+final MockFlowFile notMatched =
testRunner.getFlowFilesForRelationship(GrokParser.REL_NO_MATCH).get(0);
+notMatched.assertContentEquals(GROK_TEXT_INPUT);
+
+}
+
+@Test(expected = java.lang.AssertionError.class)
--- End diff --
Rather than expected an AssertionError, we should avoid calling
testRunner.run() and instead just use testRunner.assertNotValid()
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
> URL: https://issues.apache.org/jira/browse/NIFI-2565
> Project: Apache NiFi
> Issue Type: Improvement
>Reporter: Andre
> Fix For: 1.1.0
>
>
> Following up on Ryan Ward to create a Grok capable parser
> https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[ https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572103#comment-15572103 ] ASF GitHub Bot commented on NIFI-2565: -- Github user markap14 commented on a diff in the pull request: https://github.com/apache/nifi/pull/1108#discussion_r83229592 --- Diff: nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestGrokParser/patterns --- @@ -0,0 +1,108 @@ +# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns --- End diff -- We have to ensure that we have proper licensing for these test files. > NiFi processor to parse logs using Grok patterns > > > Key: NIFI-2565 > URL: https://issues.apache.org/jira/browse/NIFI-2565 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Andre > Fix For: 1.1.0 > > > Following up on Ryan Ward to create a Grok capable parser > https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572094#comment-15572094
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83228313
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
+
+
+public static final String DESTINATION_ATTRIBUTE =
"flowfile-attribute";
+public static final String DESTINATION_CONTENT = "flowfile-content";
+private static final String APPLICATION_JSON = "application/json";
+
+public static final PropertyDescriptor GROK_EXPRESSION = new
PropertyDescriptor
+.Builder().name("Grok Expression")
+.description("Grok expression")
+.required(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
--- End diff --
We should probably use a custom validator to make sure that the configured
value is valid.
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
> URL: https://issues.apache.org/jira/browse/NIFI-2565
> Project: Apache NiFi
> Issue Type: Improvement
>Reporter: Andre
> Fix For: 1.1.0
>
>
> Following up on Ryan Ward to create a Grok capable parser
> https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E
--
T
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572090#comment-15572090
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83226928
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
--- End diff --
No need for the @SeeAlso, @ReadsAtributes, and @WritesAttributes
annotations if they are not being used.
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
> URL: https://issues.apache.org/jira/browse/NIFI-2565
> Project: Apache NiFi
> Issue Type: Improvement
>Reporter: Andre
> Fix For: 1.1.0
>
>
> Following up on Ryan Ward to create a Grok capable parser
> https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572088#comment-15572088
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83226308
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
--- End diff --
The naming convention that we try to stick with for Processors is
. While this may be counter-intuitive for a Java Developer, it
results in making the flow much more readable for users. So we should consider
ParseLog or GrokLog.
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
> URL: https://issues.apache.org/jira/browse/NIFI-2565
> Project: Apache NiFi
> Issue Type: Improvement
>Reporter: Andre
> Fix For: 1.1.0
>
>
> Following up on Ryan Ward to create a Grok capable parser
> https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[GitHub] nifi pull request #1108: NIFI-2565: add Grok parser
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83228313
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
+
+
+public static final String DESTINATION_ATTRIBUTE =
"flowfile-attribute";
+public static final String DESTINATION_CONTENT = "flowfile-content";
+private static final String APPLICATION_JSON = "application/json";
+
+public static final PropertyDescriptor GROK_EXPRESSION = new
PropertyDescriptor
+.Builder().name("Grok Expression")
+.description("Grok expression")
+.required(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
--- End diff --
We should probably use a custom validator to make sure that the configured
value is valid.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] nifi pull request #1108: NIFI-2565: add Grok parser
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83229023
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
+
+
+public static final String DESTINATION_ATTRIBUTE =
"flowfile-attribute";
+public static final String DESTINATION_CONTENT = "flowfile-content";
+private static final String APPLICATION_JSON = "application/json";
+
+public static final PropertyDescriptor GROK_EXPRESSION = new
PropertyDescriptor
+.Builder().name("Grok Expression")
+.description("Grok expression")
+.required(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor GROK_PATTERN_FILE = new
PropertyDescriptor
+.Builder().name("Grok Pattern file")
+.description("Grok Pattern file definition")
+.required(false)
+.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor DESTINATION = new
PropertyDescriptor.Builder()
+.name("Destination")
+.description("Control if Grok output value is written as a new
flowfile attribute " +
+"or written in the flowfile content. Writing to
flowfile content will overwrite any " +
+"existing flowfile content.")
+.required(true)
+.allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
+.def
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572098#comment-15572098
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83229352
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGrokParser.java
---
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Created by snamsi on 05/10/16.
+ */
+public class TestGrokParser {
+
+private TestRunner testRunner;
+final static Path GROK_LOG_INPUT =
Paths.get("src/test/resources/TestGrokParser/apache.log");
+final static Path GROK_TEXT_INPUT =
Paths.get("src/test/resources/TestGrokParser/simple_text.log");
+
+
+@Before
+public void init() {
+testRunner = TestRunners.newTestRunner(GrokParser.class);
+}
+
+@Test
+public void testGrokParserWithMatchedContent() throws IOException {
+
+
+testRunner.setProperty(GrokParser.GROK_EXPRESSION,
"%{COMMONAPACHELOG}");
+testRunner.setProperty(GrokParser.GROK_PATTERN_FILE,
"src/test/resources/TestGrokParser/patterns");
+testRunner.enqueue(GROK_LOG_INPUT);
+testRunner.run();
+testRunner.assertAllFlowFilesTransferred(GrokParser.REL_MATCH);
+final MockFlowFile matched =
testRunner.getFlowFilesForRelationship(GrokParser.REL_MATCH).get(0);
+
+matched.assertAttributeEquals("verb","GET");
+matched.assertAttributeEquals("response","401");
+matched.assertAttributeEquals("bytes","12846");
+matched.assertAttributeEquals("clientip","64.242.88.10");
+matched.assertAttributeEquals("auth","-");
+matched.assertAttributeEquals("timestamp","07/Mar/2004:16:05:49
-0800");
+
matched.assertAttributeEquals("request","/twiki/bin/edit/Main/Double_bounce_sender?topicparent=Main.ConfigurationVariables");
+matched.assertAttributeEquals("httpversion","1.1");
+
+}
+
+@Test
+public void testGrokParserWithUnMatchedContent() throws IOException {
+
+
+testRunner.setProperty(GrokParser.GROK_EXPRESSION, "%{ADDRESS}");
+testRunner.setProperty(GrokParser.GROK_PATTERN_FILE,
"src/test/resources/TestGrokParser/patterns");
+testRunner.enqueue(GROK_TEXT_INPUT);
+testRunner.run();
+testRunner.assertAllFlowFilesTransferred(GrokParser.REL_NO_MATCH);
+final MockFlowFile notMatched =
testRunner.getFlowFilesForRelationship(GrokParser.REL_NO_MATCH).get(0);
+notMatched.assertContentEquals(GROK_TEXT_INPUT);
+
+}
+
+@Test(expected = java.lang.AssertionError.class)
+public void testGrokParserWithNotFoundPatternFile() throws IOException
{
+
+testRunner.setProperty(GrokParser.GROK_EXPRESSION,
"%{COMMONAPACHELOG}");
+testRunner.setProperty(GrokParser.GROK_PATTERN_FILE,
"src/test/resources/TestGrokParser/toto_file");
+testRunner.enqueue(GROK_LOG_INPUT);
+testRunner.run();
+
+}
+
+
+@Test(expected = java.lang.AssertionError.class)
--- End diff --
Rather than expected an AssertionError, we should avoid calling
testRunner.run() and instead just use testRunner.assertNotValid()
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
>
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572087#comment-15572087
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83226436
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
--- End diff --
We should probably expand on this a bit more. Many users will not know what
Grok is.
> NiFi processor to parse logs using Grok patterns
>
>
> Key: NIFI-2565
> URL: https://issues.apache.org/jira/browse/NIFI-2565
> Project: Apache NiFi
> Issue Type: Improvement
>Reporter: Andre
> Fix For: 1.1.0
>
>
> Following up on Ryan Ward to create a Grok capable parser
> https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572095#comment-15572095
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83227771
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
+
+
+public static final String DESTINATION_ATTRIBUTE =
"flowfile-attribute";
+public static final String DESTINATION_CONTENT = "flowfile-content";
+private static final String APPLICATION_JSON = "application/json";
+
+public static final PropertyDescriptor GROK_EXPRESSION = new
PropertyDescriptor
+.Builder().name("Grok Expression")
+.description("Grok expression")
+.required(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor GROK_PATTERN_FILE = new
PropertyDescriptor
+.Builder().name("Grok Pattern file")
+.description("Grok Pattern file definition")
+.required(false)
+.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor DESTINATION = new
PropertyDescriptor.Builder()
+.name("Destination")
+.description("Control if Grok output value is written as a new
flowfile attribute " +
+"or written in the flow
[GitHub] nifi pull request #1108: NIFI-2565: add Grok parser
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83226835
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
--- End diff --
We should consider several more tags: grok, log, text, parse, delimit,
extract
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[
https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572093#comment-15572093
]
ASF GitHub Bot commented on NIFI-2565:
--
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/1108#discussion_r83228132
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GrokParser.java
---
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.flowfile.FlowFile;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Collections;
+
+
+@Tags({"Grok Processor"})
+@CapabilityDescription("Use Grok expression ,a la logstash, to parse
data.")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class GrokParser extends AbstractProcessor {
+
+
+public static final String DESTINATION_ATTRIBUTE =
"flowfile-attribute";
+public static final String DESTINATION_CONTENT = "flowfile-content";
+private static final String APPLICATION_JSON = "application/json";
+
+public static final PropertyDescriptor GROK_EXPRESSION = new
PropertyDescriptor
+.Builder().name("Grok Expression")
+.description("Grok expression")
+.required(true)
+.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor GROK_PATTERN_FILE = new
PropertyDescriptor
+.Builder().name("Grok Pattern file")
+.description("Grok Pattern file definition")
+.required(false)
+.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor DESTINATION = new
PropertyDescriptor.Builder()
+.name("Destination")
+.description("Control if Grok output value is written as a new
flowfile attribute " +
+"or written in the flow
[jira] [Commented] (NIFI-2565) NiFi processor to parse logs using Grok patterns
[ https://issues.apache.org/jira/browse/NIFI-2565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15572102#comment-15572102 ] ASF GitHub Bot commented on NIFI-2565: -- Github user markap14 commented on a diff in the pull request: https://github.com/apache/nifi/pull/1108#discussion_r83229754 --- Diff: nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestGrokParser/apache.log --- @@ -0,0 +1 @@ +64.242.88.10 - - [07/Mar/2004:16:05:49 -0800] "GET /twiki/bin/edit/Main/Double_bounce_sender?topicparent=Main.ConfigurationVariables HTTP/1.1" 401 12846 --- End diff -- We have to ensure that we have proper licensing for these test files. This one may be one that you created yourself? If not, we need to ensure that its license is properly accounted for - or just mock out a new one. > NiFi processor to parse logs using Grok patterns > > > Key: NIFI-2565 > URL: https://issues.apache.org/jira/browse/NIFI-2565 > Project: Apache NiFi > Issue Type: Improvement >Reporter: Andre > Fix For: 1.1.0 > > > Following up on Ryan Ward to create a Grok capable parser > https://mail-archives.apache.org/mod_mbox/nifi-dev/201606.mbox/%3CCADD=rnPa8nHkJbeM280=PTQ=wurtwhstm5u+7btoo9pcym2...@mail.gmail.com%3E -- This message was sent by Atlassian JIRA (v6.3.4#6332)
