[
https://issues.apache.org/jira/browse/NIFI-4789?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16343711#comment-16343711
]
ASF GitHub Bot commented on NIFI-4789:
--------------------------------------
Github user markap14 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/2411#discussion_r164499345
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractGrok.java
---
@@ -209,78 +287,94 @@ public void onTrigger(final ProcessContext context,
final ProcessSession session
try {
final byte[] byteBuffer = buffer;
- session.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- StreamUtils.fillBuffer(in, byteBuffer, false);
- }
- });
+ session.read(flowFile, in -> StreamUtils.fillBuffer(in,
byteBuffer, false));
final long len = Math.min(byteBuffer.length,
flowFile.getSize());
contentString = new String(byteBuffer, 0, (int) len, charset);
} finally {
bufferQueue.offer(buffer);
}
- final Match gm = grok.match(contentString);
- gm.captures();
-
- if (gm.toMap().isEmpty()) {
- session.transfer(flowFile, REL_NO_MATCH);
- getLogger().info("Did not match any Grok Expressions for
FlowFile {}", new Object[]{flowFile});
- return;
- }
-
- final ObjectMapper objectMapper = new ObjectMapper();
- switch (context.getProperty(DESTINATION).getValue()) {
- case FLOWFILE_ATTRIBUTE:
- Map<String, String> grokResults = new HashMap<>();
- for (Map.Entry<String, Object> entry :
gm.toMap().entrySet()) {
- if (null != entry.getValue()) {
- grokResults.put("grok." + entry.getKey(),
entry.getValue().toString());
+ try{
+ for (Grok grok : grokList) {
+ final Match gm = grok.match(contentString);
+ gm.captures();
+ final Map<String, Object> localResults = gm.toMap();
+ if (!localResults.isEmpty()) {
+
matchedExpressionList.add(grok.getOriginalGrokPattern());
+ results.putAll(localResults);
+ if (breakOnFirstMatch) {
+ break;
}
}
+ }
- flowFile = session.putAllAttributes(flowFile, grokResults);
- session.getProvenanceReporter().modifyAttributes(flowFile);
- session.transfer(flowFile, REL_MATCH);
- getLogger().info("Matched {} Grok Expressions and added
attributes to FlowFile {}", new Object[]{grokResults.size(), flowFile});
-
- break;
- case FLOWFILE_CONTENT:
- FlowFile conFlowfile = session.write(flowFile, new
StreamCallback() {
- @Override
- public void process(InputStream in, OutputStream out)
throws IOException {
-
out.write(objectMapper.writeValueAsBytes(gm.toMap()));
- }
- });
- conFlowfile = session.putAttribute(conFlowfile,
CoreAttributes.MIME_TYPE.key(), APPLICATION_JSON);
- session.getProvenanceReporter().modifyContent(conFlowfile,
"Replaced content with parsed Grok fields and values",
stopWatch.getElapsed(TimeUnit.MILLISECONDS));
- session.transfer(conFlowfile, REL_MATCH);
+ if (results.isEmpty()) {
+ session.transfer(flowFile, REL_NO_MATCH);
+ getLogger().debug("Did not match any Grok Expression for
FlowFile {}", new Object[]{flowFile});
+ return;
+ }
+
+ String matchedExpressions =
StringUtils.join(matchedExpressionList, expressionSeparator);
+ flowFile = session.putAttribute(flowFile,
matchedExpressionAttribute, matchedExpressions);
- break;
+ switch (context.getProperty(DESTINATION).getValue()) {
+ case FLOWFILE_ATTRIBUTE:
+ Map<String, String> grokResults = new HashMap<>();
+ for (Map.Entry<String, Object> entry :
results.entrySet()) {
+ if (null != entry.getValue()) {
+ grokResults.put(resultPrefix + entry.getKey(),
entry.getValue().toString());
+ }
+ }
+ flowFile = session.putAllAttributes(flowFile,
grokResults);
+
session.getProvenanceReporter().modifyAttributes(flowFile);
+ session.transfer(flowFile, REL_MATCH);
+ getLogger().info("Matched {} Grok Expressions and
added attributes to FlowFile {}", new Object[]{grokResults.size(), flowFile});
+ break;
+ case FLOWFILE_CONTENT:
+ final ObjectMapper objectMapper = new ObjectMapper();
+ FlowFile conFlowfile = session.write(flowFile, (in,
out) -> {
+ out.write(objectMapper.writeValueAsBytes(results));
+ });
+
+ conFlowfile = session.putAttribute(conFlowfile,
CoreAttributes.MIME_TYPE.key(), APPLICATION_JSON);
+
session.getProvenanceReporter().modifyContent(conFlowfile, "Replaced content
with parsed Grok fields and values",
stopWatch.getElapsed(TimeUnit.MILLISECONDS));
+ session.transfer(conFlowfile, REL_MATCH);
+ break;
+ }
+ }catch(ProcessException t){
+ flowFile = session.putAttribute(flowFile,
getClass().getSimpleName() + ".exception", t.getMessage());
+ session.transfer(flowFile, REL_NO_MATCH);
+ getLogger().error("Did not match any Grok Expression for
FlowFile {}", new Object[]{flowFile});
+ return;
}
}
+ public static Validator validateMultipleFilesExist() {
+ return (subject, input, context) -> {
+ for (String s : input.split(PATTERN_FILE_LIST_SEPARATOR)) {
--- End diff --
The logic here doesn't appear to be right. This is going to create a
validator for the first file that it encounters but not validate the others.
> Enhance ExtractGrok processor to handle multiple grok expressions
> -----------------------------------------------------------------
>
> Key: NIFI-4789
> URL: https://issues.apache.org/jira/browse/NIFI-4789
> Project: Apache NiFi
> Issue Type: New Feature
> Components: Core Framework
> Affects Versions: 1.2.0, 1.5.0
> Environment: all
> Reporter: Charles Porter
> Priority: Minor
> Labels: features
>
> Many flows require running several grok expressions against an input to
> correctly tag and extract data. using many separate grok processors to
> accomplish this is unwieldy and hard to maintain. Supporting multiple grok
> expressions delimited by comma or user selected delimiter greatly simplifies
> this.
> Feature is coded and tested, ready for pull request, if feature is approved
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
