This is an automated email from the ASF dual-hosted git repository.
mattyb149 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 48a6c81 NIFI-4735: ParseEVTX only outputs one event per chunk
48a6c81 is described below
commit 48a6c81fa261339c645773a3124ce0ee351346c4
Author: Ferenc Szabó <[email protected]>
AuthorDate: Wed Mar 20 12:28:08 2019 +0100
NIFI-4735: ParseEVTX only outputs one event per chunk
This change is based on https://github.com/apache/nifi/pull/2489
I have reproduced the issue with some additional test cases and test files
then applied the original fix.
commit message from the original change:
Updated the EVTX FileHeader class to correctly check if there are more
chunks in the file. Previously this would not process the last chunk.
Updated the EVTX ChunkHeader class to correctly check if there are
additional records in the chunk. Previously this would only process the first
record of each chunk. It was using the fileLastRecordNumber where it should
have been using the logLastRecordNumber value.
Updated the EVTX unit tests to have the correct expected number of events
and use the logLastRecordNumber.
refactoring duplicated code and magic numbers
Signed-off-by: Matthew Burgess <[email protected]>
This closes #2489
This closes #3379
---
.../nifi/processors/evtx/parser/ChunkHeader.java | 2 +-
.../nifi/processors/evtx/parser/FileHeader.java | 4 +-
.../apache/nifi/processors/evtx/ParseEvtxTest.java | 43 ++++++++++++++++++---
.../processors/evtx/parser/ChunkHeaderTest.java | 4 +-
.../src/test/resources/1344_events.evtx | Bin 0 -> 1118208 bytes
.../test/resources/3778_events_not_exported.evtx | Bin 0 -> 1052672 bytes
6 files changed, 43 insertions(+), 10 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/ChunkHeader.java
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/ChunkHeader.java
index 7f01adf..bb4e4d7 100644
---
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/ChunkHeader.java
+++
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/ChunkHeader.java
@@ -158,7 +158,7 @@ public class ChunkHeader extends Block {
}
public boolean hasNext() {
- return fileLastRecordNumber.compareTo(recordNumber) > 0;
+ return logLastRecordNumber.compareTo(recordNumber) > 0;
}
public String getString(int offset) {
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/FileHeader.java
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/FileHeader.java
index 8610fe9..914d518 100644
---
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/FileHeader.java
+++
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/main/java/org/apache/nifi/processors/evtx/parser/FileHeader.java
@@ -141,10 +141,10 @@ public class FileHeader extends Block {
/**
* Tests whether there are more chunks
- * @return true iff there are chunks left
+ * @return true if there are chunks left
*/
public boolean hasNext() {
- return count < chunkCount;
+ return count <= chunkCount;
}
/**
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
index 2e5e90d..260869d 100644
---
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
+++
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
@@ -74,6 +74,7 @@ public class ParseEvtxTest {
public static final String USER_DATA = "UserData";
public static final String EVENT_DATA = "EventData";
public static final Set DATA_TAGS = new
HashSet<>(Arrays.asList(EVENT_DATA, USER_DATA));
+ public static final int EXPECTED_SUCCESSFUL_EVENT_COUNT = 1053;
@Mock
FileHeaderFactory fileHeaderFactory;
@@ -366,7 +367,7 @@ public class ParseEvtxTest {
assertEquals(1, failureFlowFiles.size());
validateFlowFiles(failureFlowFiles);
// We expect the same number of records to come out no matter the
granularity
- assertEquals(960, validateFlowFiles(failureFlowFiles));
+ assertEquals(EXPECTED_SUCCESSFUL_EVENT_COUNT,
validateFlowFiles(failureFlowFiles));
// Whole file fails if there is a failure parsing
List<MockFlowFile> successFlowFiles =
testRunner.getFlowFilesForRelationship(ParseEvtx.REL_SUCCESS);
@@ -399,10 +400,10 @@ public class ParseEvtxTest {
assertEquals(1, failureFlowFiles.size());
List<MockFlowFile> successFlowFiles =
testRunner.getFlowFilesForRelationship(ParseEvtx.REL_SUCCESS);
- assertEquals(8, successFlowFiles.size());
+ assertEquals(9, successFlowFiles.size());
// We expect the same number of records to come out no matter the
granularity
- assertEquals(960, validateFlowFiles(successFlowFiles) +
validateFlowFiles(failureFlowFiles));
+ assertEquals(EXPECTED_SUCCESSFUL_EVENT_COUNT,
validateFlowFiles(successFlowFiles) + validateFlowFiles(failureFlowFiles));
}
@Test
@@ -433,10 +434,42 @@ public class ParseEvtxTest {
// Whole file fails if there is a failure parsing
List<MockFlowFile> successFlowFiles =
testRunner.getFlowFilesForRelationship(ParseEvtx.REL_SUCCESS);
- assertEquals(960, successFlowFiles.size());
+ assertEquals(EXPECTED_SUCCESSFUL_EVENT_COUNT, successFlowFiles.size());
// We expect the same number of records to come out no matter the
granularity
- assertEquals(960, validateFlowFiles(successFlowFiles));
+ assertEquals(EXPECTED_SUCCESSFUL_EVENT_COUNT,
validateFlowFiles(successFlowFiles));
+ }
+
+ @Test
+ public void testRecordBasedParseCorrectNumberOfFlowFiles() {
+ testValidEvents(ParseEvtx.RECORD, "1344_events.evtx", 1344);
+ }
+
+ @Test
+ public void testChunkBasedParseCorrectNumberOfFlowFiles() {
+ testValidEvents(ParseEvtx.CHUNK, "1344_events.evtx", 14);
+ }
+
+ @Test
+ public void testRecordBasedParseCorrectNumberOfFlowFilesFromAResizedFile()
{
+ testValidEvents(ParseEvtx.RECORD, "3778_events_not_exported.evtx",
3778);
+ }
+
+ @Test
+ public void testChunkBasedParseCorrectNumberOfFlowFilesFromAResizedFile() {
+ testValidEvents(ParseEvtx.CHUNK, "3778_events_not_exported.evtx", 16);
+ }
+
+ private void testValidEvents(String granularity, String filename, int
expectedCount) {
+ TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
+ testRunner.setProperty(ParseEvtx.GRANULARITY, granularity);
+ Map<String, String> attributes = new HashMap<>();
+ ClassLoader classLoader = this.getClass().getClassLoader();
+ InputStream resourceAsStream =
classLoader.getResourceAsStream(filename);
+ testRunner.enqueue(resourceAsStream, attributes);
+ testRunner.run();
+
+ testRunner.assertTransferCount(ParseEvtx.REL_SUCCESS, expectedCount);
}
private int validateFlowFiles(List<MockFlowFile> successFlowFiles) throws
SAXException, IOException, ParserConfigurationException {
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/parser/ChunkHeaderTest.java
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/parser/ChunkHeaderTest.java
index 1cb52db..247d37f 100644
---
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/parser/ChunkHeaderTest.java
+++
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/parser/ChunkHeaderTest.java
@@ -102,7 +102,7 @@ public class ChunkHeaderTest {
offset += 11;
}
- RecordTest.putNode(testBinaryReaderBuilder, fileLastRecordNumber, new
Date());
+ RecordTest.putNode(testBinaryReaderBuilder, logLastRecordNumber, new
Date());
testBinaryReaderBuilder.put(dataBuilder.toByteArray());
@@ -133,7 +133,7 @@ public class ChunkHeaderTest {
assertTrue(chunkHeader.hasNext());
Record next = chunkHeader.next();
- assertEquals(fileLastRecordNumber, next.getRecordNum().intValue());
+ assertEquals(logLastRecordNumber, next.getRecordNum().intValue());
RootNode rootNode = next.getRootNode();
List<BxmlNode> children = rootNode.getChildren();
assertEquals(1, children.size());
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/1344_events.evtx
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/1344_events.evtx
new file mode 100644
index 0000000..31a084d
Binary files /dev/null and
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/1344_events.evtx
differ
diff --git
a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/3778_events_not_exported.evtx
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/3778_events_not_exported.evtx
new file mode 100644
index 0000000..1110090
Binary files /dev/null and
b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/resources/3778_events_not_exported.evtx
differ
