This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new bb8ea06 MINIFICPP-1448 CWEL JSON output
bb8ea06 is described below
commit bb8ea0632d50d9ac43e634847ff13c0955bc43ff
Author: Adam Debreceni <[email protected]>
AuthorDate: Sat Jan 23 21:42:30 2021 +0100
MINIFICPP-1448 CWEL JSON output
This closes #976
Signed-off-by: Marton Szasz <[email protected]>
Co-authored-by: Marton Szasz <[email protected]>
---
.github/workflows/ci.yml | 10 +-
.../windows-event-log/ConsumeWindowsEventLog.cpp | 132 +++++++++++-----
.../windows-event-log/ConsumeWindowsEventLog.h | 40 +++--
extensions/windows-event-log/tests/CMakeLists.txt | 10 +-
.../tests/CWELCustomProviderTests.cpp | 176 +++++++++++++++++++++
extensions/windows-event-log/tests/CWELTestUtils.h | 117 ++++++++++++++
.../tests/ConsumeWindowsEventLogTests.cpp | 127 ++++++++++-----
.../custom-provider/generate-and-register.bat | 62 ++++++++
extensions/windows-event-log/wel/JSONUtils.cpp | 170 ++++++++++++++++++++
extensions/windows-event-log/wel/JSONUtils.h | 86 ++++++++++
.../windows-event-log/wel/MetadataWalker.cpp | 5 +-
.../windows-event-log/wel/WindowsEventLog.cpp | 5 +
win_build_vs.bat | 5 +-
13 files changed, 846 insertions(+), 99 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1e00d42..69e8717 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,7 +47,10 @@ jobs:
- name: Setup PATH
uses: microsoft/[email protected]
- id: build
- run: win_build_vs.bat build /CI /S /A
+ run: |
+ PATH %PATH%;C:\Program Files (x86)\Windows
Kits\10\bin\10.0.17763.0\x86
+ PATH %PATH%;C:\Program Files (x86)\Microsoft Visual
Studio\2017\Enterprise\MSBuild\15.0\Bin\Roslyn
+ win_build_vs.bat build /CI /S /A
shell: cmd
windows_VS2019:
name: "windows-vs2019"
@@ -59,7 +62,10 @@ jobs:
- name: Setup PATH
uses: microsoft/[email protected]
- id: build
- run: win_build_vs.bat build /2019 /64 /CI
+ run: |
+ PATH %PATH%;C:\Program Files (x86)\Windows
Kits\10\bin\10.0.19041.0\x64
+ PATH %PATH%;C:\Program Files (x86)\Microsoft Visual
Studio\2019\Enterprise\MSBuild\Current\Bin\Roslyn
+ win_build_vs.bat build /2019 /64 /CI
shell: cmd
ubuntu_16_04:
name: "ubuntu-16.04"
diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
index 22442f9..f262402 100644
--- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
+++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
@@ -34,6 +34,7 @@
#include "wel/MetadataWalker.h"
#include "wel/XMLString.h"
#include "wel/UnicodeConversion.h"
+#include "wel/JSONUtils.h"
#include "io/BufferStream.h"
#include "core/ProcessContext.h"
@@ -134,8 +135,16 @@ core::Property ConsumeWindowsEventLog::OutputFormat(
core::PropertyBuilder::createProperty("Output Format")->
isRequired(true)->
withDefaultValue(Both)->
- withAllowableValues<std::string>({XML, Plaintext, Both})->
- withDescription("Set the output format type. In case \'Both\' is selected
the processor generates two flow files for every event captured")->
+ withAllowableValues<std::string>({XML, Plaintext, Both, JSON})->
+ withDescription("Set the output format type. In case \'Both\' is selected
the processor generates two flow files for every event captured in format XML
and Plaintext")->
+ build());
+
+core::Property ConsumeWindowsEventLog::JSONFormat(
+ core::PropertyBuilder::createProperty("JSON Format")->
+ isRequired(true)->
+ withDefaultValue(JSONSimple)->
+ withAllowableValues<std::string>({JSONSimple, JSONFlattened, JSONRaw})->
+ withDescription("Set the json format type. Only applicable if Output Format
is set to 'JSON'")->
build());
core::Property ConsumeWindowsEventLog::BatchCommitSize(
@@ -162,7 +171,8 @@ core::Property ConsumeWindowsEventLog::ProcessOldEvents(
core::Relationship ConsumeWindowsEventLog::Success("success", "Relationship
for successfully consumed events.");
ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name,
utils::Identifier uuid)
- : core::Processor(name, uuid),
logger_(logging::LoggerFactory<ConsumeWindowsEventLog>::getLogger()),
apply_identifier_function_(false), batch_commit_size_(0U) {
+ : core::Processor(name, uuid),
+ logger_(logging::LoggerFactory<ConsumeWindowsEventLog>::getLogger()) {
char buff[MAX_COMPUTERNAME_LENGTH + 1];
DWORD size = sizeof(buff);
if (GetComputerName(buff, &size)) {
@@ -170,9 +180,6 @@ ConsumeWindowsEventLog::ConsumeWindowsEventLog(const
std::string& name, utils::I
} else {
LogWindowsError();
}
-
- writeXML_ = false;
- writePlainText_ = false;
}
void ConsumeWindowsEventLog::notifyStop() {
@@ -199,7 +206,7 @@ void ConsumeWindowsEventLog::initialize() {
//! Set the supported properties
setSupportedProperties({
Channel, Query, MaxBufferSize, InactiveDurationToReconnect,
IdentifierMatcher, IdentifierFunction, ResolveAsAttributes,
- EventHeaderDelimiter, EventHeader, OutputFormat, BatchCommitSize,
BookmarkRootDirectory, ProcessOldEvents
+ EventHeaderDelimiter, EventHeader, OutputFormat, JSONFormat,
BatchCommitSize, BookmarkRootDirectory, ProcessOldEvents
});
//! Set the supported relationships
@@ -252,11 +259,31 @@ void ConsumeWindowsEventLog::onSchedule(const
std::shared_ptr<core::ProcessConte
std::string mode;
context->getProperty(OutputFormat.getName(), mode);
- writeXML_ = (mode == Both || mode == XML);
-
- writePlainText_ = (mode == Both || mode == Plaintext);
+ output_ = {};
+ if (mode == XML) {
+ output_.xml = true;
+ } else if (mode == Plaintext) {
+ output_.plaintext = true;
+ } else if (mode == Both) {
+ output_.xml = true;
+ output_.plaintext = true;
+ } else if (mode == JSON) {
+ std::string json_format;
+ context->getProperty(JSONFormat.getName(), json_format);
+ if (json_format == JSONRaw) {
+ output_.json.type = JSONType::Raw;
+ } else if (json_format == JSONSimple) {
+ output_.json.type = JSONType::Simple;
+ } else if (json_format == JSONFlattened) {
+ output_.json.type = JSONType::Flattened;
+ }
+ } else {
+ // in the future this might be considered an error, but for now due to
backwards
+ // compatibility we just fall through and execute the processor outputing
nothing
+ // throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Unrecognized output
format: " + mode);
+ }
- if (writeXML_ && !hMsobjsDll_) {
+ if ((output_.xml || output_.json) && !hMsobjsDll_) {
char systemDir[MAX_PATH];
if (GetSystemDirectory(systemDir, sizeof(systemDir))) {
hMsobjsDll_ = LoadLibrary((systemDir +
std::string("\\msobjs.dll")).c_str());
@@ -564,7 +591,7 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE
hEvent, EventRender& e
logger_->log_debug("Finish doc traversing, performing writing...");
- if (writePlainText_) {
+ if (output_.plaintext) {
logger_->log_trace("Writing event in plain text");
auto handler = getEventLogHandler(providerName);
@@ -583,30 +610,47 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE
hEvent, EventRender& e
// set the delimiter
log_header.setDelimiter(header_delimiter_);
// render the header.
- eventRender.rendered_text_ =
log_header.getEventHeader([&walker](wel::METADATA metadata) { return
walker.getMetadata(metadata); });
- eventRender.rendered_text_ += "Message" + header_delimiter_ + " ";
- eventRender.rendered_text_ += message;
+ eventRender.plaintext =
log_header.getEventHeader([&walker](wel::METADATA metadata) { return
walker.getMetadata(metadata); });
+ eventRender.plaintext += "Message" + header_delimiter_ + " ";
+ eventRender.plaintext += message;
}
logger_->log_trace("Finish writing in plain text");
}
- if (writeXML_) {
- logger_->log_trace("Writing event in XML");
+ if (output_.xml || output_.json) {
substituteXMLPercentageItems(doc);
logger_->log_trace("Finish substituting %% in XML");
if (resolve_as_attributes_) {
- eventRender.matched_fields_ = walker.getFieldValues();
+ eventRender.matched_fields = walker.getFieldValues();
}
+ }
+
+ if (output_.xml) {
+ logger_->log_trace("Writing event in XML");
wel::XmlString writer;
doc.print(writer, "", pugi::format_raw); // no indentation or formatting
xml = writer.xml_;
- eventRender.text_ = std::move(xml);
+ eventRender.xml = std::move(xml);
logger_->log_trace("Finish writing in XML");
}
+ if (output_.json.type == JSONType::Raw) {
+ logger_->log_trace("Writing event in raw JSON");
+ eventRender.json = wel::jsonToString(wel::toRawJSON(doc));
+ logger_->log_trace("Finish writing in raw JSON");
+ } else if (output_.json.type == JSONType::Simple) {
+ logger_->log_trace("Writing event in simple JSON");
+ eventRender.json = wel::jsonToString(wel::toSimpleJSON(doc));
+ logger_->log_trace("Finish writing in simple JSON");
+ } else if (output_.json.type == JSONType::Flattened) {
+ logger_->log_trace("Writing event in flattened JSON");
+ eventRender.json = wel::jsonToString(wel::toFlattenedJSON(doc));
+ logger_->log_trace("Finish writing in flattened JSON");
+ }
+
return true;
}
@@ -658,39 +702,45 @@ void
ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender&
const std::string& str_;
};
- if (writeXML_) {
- auto flowFile = session.create();
- logger_->log_trace("Writing rendered XML to a flow file");
-
+ auto commitFlowFile = [&] (const std::shared_ptr<core::FlowFile>& flowFile,
const std::string& content, const std::string& mimeType) {
{
- WriteCallback wc{ eventRender.text_ };
+ WriteCallback wc{ content };
session.write(flowFile, &wc);
}
- for (const auto &fieldMapping : eventRender.matched_fields_) {
- if (!fieldMapping.second.empty()) {
- session.putAttribute(flowFile, fieldMapping.first,
fieldMapping.second);
- }
- }
- session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE,
"application/xml");
+ session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE,
mimeType);
session.putAttribute(flowFile, "Timezone name", timezone_name_);
session.putAttribute(flowFile, "Timezone offset", timezone_offset_);
session.getProvenanceReporter()->receive(flowFile, provenanceUri_,
getUUIDStr(), "Consume windows event logs", 0);
session.transfer(flowFile, Success);
- }
+ };
- if (writePlainText_) {
+ if (output_.xml) {
auto flowFile = session.create();
- logger_->log_trace("Writing rendered plain text to a flow file");
+ logger_->log_trace("Writing rendered XML to a flow file");
- {
- WriteCallback wc{ eventRender.rendered_text_ };
- session.write(flowFile, &wc);
+ for (const auto &fieldMapping : eventRender.matched_fields) {
+ if (!fieldMapping.second.empty()) {
+ session.putAttribute(flowFile, fieldMapping.first,
fieldMapping.second);
+ }
}
- session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE,
"text/plain");
- session.putAttribute(flowFile, "Timezone name", timezone_name_);
- session.putAttribute(flowFile, "Timezone offset", timezone_offset_);
- session.getProvenanceReporter()->receive(flowFile, provenanceUri_,
getUUIDStr(), "Consume windows event logs", 0);
- session.transfer(flowFile, Success);
+
+ commitFlowFile(flowFile, eventRender.xml, "application/xml");
+ }
+
+ if (output_.plaintext) {
+ logger_->log_trace("Writing rendered plain text to a flow file");
+ commitFlowFile(session.create(), eventRender.plaintext, "text/plain");
+ }
+
+ if (output_.json.type == JSONType::Raw) {
+ logger_->log_trace("Writing rendered raw JSON to a flow file");
+ commitFlowFile(session.create(), eventRender.json, "application/json");
+ } else if (output_.json.type == JSONType::Simple) {
+ logger_->log_trace("Writing rendered simple JSON to a flow file");
+ commitFlowFile(session.create(), eventRender.json, "application/json");
+ } else if (output_.json.type == JSONType::Flattened) {
+ logger_->log_trace("Writing rendered flattened JSON to a flow file");
+ commitFlowFile(session.create(), eventRender.json, "application/json");
}
}
diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h
b/extensions/windows-event-log/ConsumeWindowsEventLog.h
index db4f13b..d0820bf 100644
--- a/extensions/windows-event-log/ConsumeWindowsEventLog.h
+++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h
@@ -43,9 +43,10 @@ namespace minifi {
namespace processors {
struct EventRender {
- std::map<std::string, std::string> matched_fields_;
- std::string text_;
- std::string rendered_text_;
+ std::map<std::string, std::string> matched_fields;
+ std::string xml;
+ std::string plaintext;
+ std::string json;
};
class Bookmark;
@@ -77,6 +78,7 @@ public:
static core::Property EventHeaderDelimiter;
static core::Property EventHeader;
static core::Property OutputFormat;
+ static core::Property JSONFormat;
static core::Property BatchCommitSize;
static core::Property BookmarkRootDirectory;
static core::Property ProcessOldEvents;
@@ -107,9 +109,13 @@ protected:
bool createEventRender(EVT_HANDLE eventHandle, EventRender& eventRender);
void substituteXMLPercentageItems(pugi::xml_document& doc);
- static constexpr const char * const XML = "XML";
- static constexpr const char * const Both = "Both";
- static constexpr const char * const Plaintext = "Plaintext";
+ static constexpr const char* XML = "XML";
+ static constexpr const char* Both = "Both";
+ static constexpr const char* Plaintext = "Plaintext";
+ static constexpr const char* JSON = "JSON";
+ static constexpr const char* JSONRaw = "Raw";
+ static constexpr const char* JSONSimple = "Simple";
+ static constexpr const char* JSONFlattened = "Flattened";
private:
struct TimeDiff {
@@ -132,18 +138,30 @@ private:
std::wstring wstrChannel_;
std::wstring wstrQuery_;
std::string regex_;
- bool resolve_as_attributes_;
- bool apply_identifier_function_;
+ bool resolve_as_attributes_{false};
+ bool apply_identifier_function_{false};
std::string provenanceUri_;
std::string computerName_;
uint64_t maxBufferSize_{};
DWORD lastActivityTimestamp_{};
std::mutex cache_mutex_;
std::map<std::string, wel::WindowsEventLogHandler > providers_;
- uint64_t batch_commit_size_;
+ uint64_t batch_commit_size_{};
+
+ enum class JSONType {None, Raw, Simple, Flattened};
+
+ struct OutputFormat {
+ bool xml{false};
+ bool plaintext{false};
+ struct JSON {
+ JSONType type{JSONType::None};
+
+ explicit operator bool() const noexcept {
+ return type != JSONType::None;
+ }
+ } json;
+ } output_;
- bool writeXML_;
- bool writePlainText_;
std::unique_ptr<Bookmark> bookmark_;
std::mutex on_trigger_mutex_;
std::unordered_map<std::string, std::string> xmlPercentageItemsResolutions_;
diff --git a/extensions/windows-event-log/tests/CMakeLists.txt
b/extensions/windows-event-log/tests/CMakeLists.txt
index 4d144ce..1f8c9f3 100644
--- a/extensions/windows-event-log/tests/CMakeLists.txt
+++ b/extensions/windows-event-log/tests/CMakeLists.txt
@@ -17,7 +17,15 @@
# under the License.
#
-file(GLOB WEL_INTEGRATION_TESTS "*.cpp")
+set(WEL_INTEGRATION_TESTS "BookmarkTests.cpp"
"ConsumeWindowsEventLogTests.cpp" "MetadataWalkerTests.cpp")
+if (TEST_CUSTOM_WEL_PROVIDER)
+ execute_process(COMMAND
+
"${CMAKE_CURRENT_LIST_DIR}/custom-provider/generate-and-register.bat"
+ "${CMAKE_CURRENT_LIST_DIR}/custom-provider"
+ )
+ list(APPEND WEL_INTEGRATION_TESTS "CWELCustomProviderTests.cpp")
+endif()
+
SET(WEL_TEST_COUNT 0)
FOREACH(testfile ${WEL_INTEGRATION_TESTS})
get_filename_component(testfilename "${testfile}" NAME_WE)
diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
new file mode 100644
index 0000000..57eb4bf
--- /dev/null
+++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#undef NDEBUG
+
+#include "ConsumeWindowsEventLog.h"
+
+#include "core/ConfigurableComponent.h"
+#include "processors/LogAttribute.h"
+#include "processors/PutFile.h"
+#include "TestBase.h"
+#include "utils/TestUtils.h"
+#include "utils/file/FileUtils.h"
+#include "rapidjson/document.h"
+#include "wel/UniqueEvtHandle.h"
+#include "IntegrationTestUtils.h"
+
+#include "CWELTestUtils.h"
+
+// generated from the manifest file "custom-provider/unit-test-provider.man"
+// using the command "mc -um unit-test-provider.man"
+#include "custom-provider/unit-test-provider.h"
+
+namespace {
+
+struct CustomEventData {
+ std::wstring first;
+ std::wstring second;
+ std::wstring third;
+ int binary_length;
+ const unsigned char* binary_data;
+};
+
+const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider";
+const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log";
+
+bool dispatchCustomEvent(const CustomEventData& event) {
+ static auto provider_initialized = EventRegisterminifi_unit_test_provider();
+ REQUIRE(provider_initialized == ERROR_SUCCESS);
+
+ auto result = EventWriteCustomEvent(
+ event.first.c_str(),
+ event.second.c_str(),
+ event.third.c_str(),
+ event.binary_length,
+ event.binary_data
+ );
+ return result == ERROR_SUCCESS;
+}
+
+using org::apache::nifi::minifi::wel::unique_evt_handle;
+
+bool advanceBookmark(const unique_evt_handle& hBookmark, const std::string&
channel, const std::string& query, bool advance_to_last = false) {
+ const auto hEventResults = unique_evt_handle{ EvtQuery(0,
std::wstring{channel.begin(), channel.end()}.c_str(),
std::wstring{query.begin(), query.end()}.c_str(), EvtQueryChannelPath) };
+ if (!hEventResults) {
+ return false;
+ }
+
+ if (advance_to_last) {
+ if (!EvtSeek(hEventResults.get(), 0, 0, 0, EvtSeekRelativeToLast)) {
+ return false;
+ }
+ } else {
+ if (!EvtSeek(hEventResults.get(), 1, hBookmark.get(), 0,
EvtSeekRelativeToBookmark)) {
+ return false;
+ }
+ }
+
+ const unique_evt_handle hEvent = [&hEventResults] {
+ DWORD dwReturned{};
+ EVT_HANDLE hEvent{ nullptr };
+ EvtNext(hEventResults.get(), 1, &hEvent, INFINITE, 0, &dwReturned);
+ return unique_evt_handle{ hEvent };
+ }();
+
+ if (!hEvent) {
+ return false;
+ }
+
+ REQUIRE(EvtUpdateBookmark(hBookmark.get(), hEvent.get()));
+
+ return true;
+}
+
+class CustomProviderController : public OutputFormatTestController {
+ public:
+ CustomProviderController(std::string format, std::string json_format) :
OutputFormatTestController(CUSTOM_CHANNEL, "*", std::move(format),
std::move(json_format)) {
+ bookmark_.reset(EvtCreateBookmark(0));
+ advanceBookmark(bookmark_, channel_, query_, true);
+ REQUIRE(bookmark_);
+ }
+
+ protected:
+ void dispatchBookmarkEvent() override {
+ auto binary = reinterpret_cast<const unsigned char*>("\x0c\x10");
+ REQUIRE(dispatchCustomEvent({L"Bookmark", L"Second", L"Third", 2,
binary}));
+ REQUIRE(checkNewEventAvailable());
+ }
+ void dispatchCollectedEvent() override {
+ auto binary = reinterpret_cast<const unsigned char*>("\x09\x01");
+ REQUIRE(dispatchCustomEvent({L"Actual event", L"Second", L"Third", 2,
binary}));
+ REQUIRE(checkNewEventAvailable());
+ }
+
+ private:
+ bool checkNewEventAvailable() {
+ return
org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5},
[&] {
+ return advanceBookmark(bookmark_, channel_, query_);
+ });
+ }
+ unique_evt_handle bookmark_;
+};
+
+const std::string EVENT_DATA_JSON = R"(
+ [{
+ "Type": "Data",
+ "Content": "Actual event",
+ "Name": "param1"
+ }, {
+ "Type": "Data",
+ "Content": "Second",
+ "Name": "param2"
+ }, {
+ "Type": "Data",
+ "Content": "Third",
+ "Name": "Channel"
+ }, {
+ "Type": "Binary",
+ "Content": "0901",
+ "Name": ""
+ }]
+)";
+
+} // namespace
+
+TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly
custom provider", "[onTrigger]") {
+ std::string event = CustomProviderController{"JSON", "Simple"}.run();
+ verifyJSON(event, R"(
+ {
+ "System": {
+ "Provider": {
+ "Name": ")" + CUSTOM_PROVIDER_NAME + R"("
+ },
+ "Channel": ")" + CUSTOM_CHANNEL + R"("
+ },
+ "EventData": )" + EVENT_DATA_JSON + R"(
+ }
+ )");
+}
+
+TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly
custom provider", "[onTrigger]") {
+ std::string event = CustomProviderController{"JSON", "Flattened"}.run();
+ verifyJSON(event, R"(
+ {
+ "Name": ")" + CUSTOM_PROVIDER_NAME + R"(",
+ "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data
named "Channel" */ + R"(",
+ "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(,
+ "param1": "Actual event",
+ "param2": "Second"
+ }
+ )");
+}
diff --git a/extensions/windows-event-log/tests/CWELTestUtils.h
b/extensions/windows-event-log/tests/CWELTestUtils.h
new file mode 100644
index 0000000..3820ca5
--- /dev/null
+++ b/extensions/windows-event-log/tests/CWELTestUtils.h
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "ConsumeWindowsEventLog.h"
+#include "processors/PutFile.h"
+#include "TestBase.h"
+#include "utils/TestUtils.h"
+#include "utils/file/FileUtils.h"
+#include "utils/OptionalUtils.h"
+
+core::Relationship Success{"success", "Everything is fine"};
+
+using ConsumeWindowsEventLog =
org::apache::nifi::minifi::processors::ConsumeWindowsEventLog;
+using PutFile = org::apache::nifi::minifi::processors::PutFile;
+
+class OutputFormatTestController : public TestController {
+ public:
+ OutputFormatTestController(std::string channel, std::string query,
std::string output_format, utils::optional<std::string> json_format = {})
+ : channel_(std::move(channel)),
+ query_(std::move(query)),
+ output_format_(std::move(output_format)),
+ json_format_(std::move(json_format)) {}
+
+ std::string run() {
+ LogTestController::getInstance().setDebug<ConsumeWindowsEventLog>();
+ LogTestController::getInstance().setDebug<PutFile>();
+ std::shared_ptr<TestPlan> test_plan = createPlan();
+
+ auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog",
"cwel");
+ test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::Channel.getName(), channel_);
+ test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::Query.getName(), query_);
+ test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::OutputFormat.getName(), output_format_);
+ if (json_format_) {
+ test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::JSONFormat.getName(), json_format_.value());
+ }
+
+ auto dir = utils::createTempDir(this);
+
+ auto put_file = test_plan->addProcessor("PutFile", "putFile", Success,
true);
+ test_plan->setProperty(put_file, PutFile::Directory.getName(), dir);
+
+ {
+ dispatchBookmarkEvent();
+
+ runSession(test_plan);
+ }
+
+ test_plan->reset();
+
LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output);
+
+
+ {
+ dispatchCollectedEvent();
+
+ runSession(test_plan);
+
+ auto files = utils::file::list_dir_all(dir,
LogTestController::getInstance().getLogger<LogTestController>(), false);
+ REQUIRE(files.size() == 1);
+
+ std::ifstream file{utils::file::concat_path(files[0].first,
files[0].second)};
+ return {std::istreambuf_iterator<char>{file}, {}};
+ }
+ }
+
+ protected:
+ virtual void dispatchBookmarkEvent() = 0;
+ virtual void dispatchCollectedEvent() = 0;
+
+ std::string channel_;
+ std::string query_;
+ std::string output_format_;
+ utils::optional<std::string> json_format_;
+};
+
+// carries out a loose match on objects, i.e. it doesn't matter if the
+// actual object has extra fields than expected
+void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected)
{
+ if (expected.IsObject()) {
+ REQUIRE(json.IsObject());
+ for (const auto& expected_member : expected.GetObject()) {
+ REQUIRE(json.HasMember(expected_member.name));
+ matchJSON(json[expected_member.name], expected_member.value);
+ }
+ } else if (expected.IsArray()) {
+ REQUIRE(json.IsArray());
+ REQUIRE(json.Size() == expected.Size());
+ for (size_t idx{0}; idx < expected.Size(); ++idx) {
+ matchJSON(json[idx], expected[idx]);
+ }
+ } else {
+ REQUIRE(json == expected);
+ }
+}
+
+void verifyJSON(const std::string& json_str, const std::string& expected_str) {
+ rapidjson::Document json, expected;
+ REQUIRE_FALSE(json.Parse(json_str.c_str()).HasParseError());
+ REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError());
+
+ matchJSON(json, expected);
+}
diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
index 02aae26..1976f40 100644
--- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
+++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
@@ -19,17 +19,22 @@
#include "core/ConfigurableComponent.h"
#include "processors/LogAttribute.h"
+#include "processors/PutFile.h"
#include "TestBase.h"
+#include "utils/TestUtils.h"
+#include "utils/file/FileUtils.h"
+#include "rapidjson/document.h"
+
+#include "CWELTestUtils.h"
using ConsumeWindowsEventLog =
org::apache::nifi::minifi::processors::ConsumeWindowsEventLog;
using LogAttribute = org::apache::nifi::minifi::processors::LogAttribute;
+using PutFile = org::apache::nifi::minifi::processors::PutFile;
using ConfigurableComponent =
org::apache::nifi::minifi::core::ConfigurableComponent;
using IdGenerator = org::apache::nifi::minifi::utils::IdGenerator;
namespace {
-core::Relationship Success{"success", "Everything is fine"};
-
const std::string APPLICATION_CHANNEL = "Application";
constexpr DWORD CWEL_TESTS_OPCODE = 14985; // random opcode hopefully won't
clash with something important
@@ -41,6 +46,19 @@ void reportEvent(const std::string& channel, const char*
message, WORD log_level
ReportEventA(event_source, log_level, 0, CWEL_TESTS_OPCODE, nullptr, 1, 0,
&message, nullptr);
}
+class SimpleFormatTestController : public OutputFormatTestController {
+ public:
+ using OutputFormatTestController::OutputFormatTestController;
+
+ protected:
+ void dispatchBookmarkEvent() {
+ reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past");
+ }
+ void OutputFormatTestController::dispatchCollectedEvent() {
+ reportEvent(APPLICATION_CHANNEL, "Event one");
+ }
+};
+
} // namespace
TEST_CASE("ConsumeWindowsEventLog constructor works", "[create]") {
@@ -305,48 +323,75 @@ TEST_CASE("ConsumeWindowsEventLog output format can be
set", "[create][output_fo
// TEST_CASE("ConsumeWindowsEventLog prints events in plain text correctly",
"[onTrigger]")
TEST_CASE("ConsumeWindowsEventLog prints events in XML correctly",
"[onTrigger]") {
- TestController test_controller;
- LogTestController::getInstance().setDebug<ConsumeWindowsEventLog>();
- LogTestController::getInstance().setDebug<LogAttribute>();
- std::shared_ptr<TestPlan> test_plan = test_controller.createPlan();
-
- auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog",
"cwel");
- test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::Channel.getName(), APPLICATION_CHANNEL);
- test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::Query.getName(), QUERY);
- test_plan->setProperty(cwel_processor,
ConsumeWindowsEventLog::OutputFormat.getName(), "XML");
-
- auto logger_processor = test_plan->addProcessor("LogAttribute", "logger",
Success, true);
- test_plan->setProperty(logger_processor,
LogAttribute::FlowFilesToLog.getName(), "0");
- test_plan->setProperty(logger_processor, LogAttribute::LogPayload.getName(),
"true");
- test_plan->setProperty(logger_processor,
LogAttribute::MaxPayloadLineLength.getName(), "1024");
-
- {
- reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past");
-
- test_controller.runSession(test_plan);
- }
-
- test_plan->reset();
-
LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output);
+ std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, QUERY,
"XML"}.run();
+
+ REQUIRE(event.find(R"(<Event
xmlns="http://schemas.microsoft.com/win/2004/08/events/event";><System><Provider
Name="Application"/>)") != std::string::npos);
+ REQUIRE(event.find(R"(<EventID Qualifiers="0">14985</EventID>)") !=
std::string::npos);
+ REQUIRE(event.find(R"(<Level>4</Level>)") != std::string::npos);
+ REQUIRE(event.find(R"(<Task>0</Task>)") != std::string::npos);
+ REQUIRE(event.find(R"(<Keywords>0x80000000000000</Keywords><TimeCreated
SystemTime=")") != std::string::npos);
+ // the timestamp (when the event was published) goes here
+ REQUIRE(event.find(R"("/><EventRecordID>)") != std::string::npos);
+ // the ID of the event goes here (a number)
+ REQUIRE(event.find(R"(</EventRecordID>)") != std::string::npos);
+ REQUIRE(event.find(R"(<Channel>Application</Channel><Computer>)") !=
std::string::npos);
+ // the computer name goes here
+ REQUIRE(event.find(R"(</Computer><Security/></System><EventData><Data>Event
one</Data></EventData></Event>)") != std::string::npos);
+}
- {
- reportEvent(APPLICATION_CHANNEL, "Event one");
+TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly",
"[onTrigger]") {
+ std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*",
"JSON", "Simple"}.run();
+ verifyJSON(event, R"json(
+ {
+ "System": {
+ "Provider": {
+ "Name": "Application"
+ },
+ "Channel": "Application"
+ },
+ "EventData": [{
+ "Type": "Data",
+ "Content": "Event one",
+ "Name": ""
+ }]
+ }
+ )json");
+}
- test_controller.runSession(test_plan);
+TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly",
"[onTrigger]") {
+ std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*",
"JSON", "Flattened"}.run();
+ verifyJSON(event, R"json(
+ {
+ "Name": "Application",
+ "Channel": "Application",
+ "EventData": [{
+ "Type": "Data",
+ "Content": "Event one",
+ "Name": ""
+ }]
+ }
+ )json");
+}
- REQUIRE(LogTestController::getInstance().contains(R"(<Event
xmlns="http://schemas.microsoft.com/win/2004/08/events/event";><System><Provider
Name="Application"/>)"));
- REQUIRE(LogTestController::getInstance().contains(R"(<EventID
Qualifiers="0">14985</EventID>)"));
- REQUIRE(LogTestController::getInstance().contains(R"(<Level>4</Level>)"));
- REQUIRE(LogTestController::getInstance().contains(R"(<Task>0</Task>)"));
-
REQUIRE(LogTestController::getInstance().contains(R"(<Keywords>0x80000000000000</Keywords><TimeCreated
SystemTime=")"));
- // the timestamp (when the event was published) goes here
-
REQUIRE(LogTestController::getInstance().contains(R"("/><EventRecordID>)"));
- // the ID of the event goes here (a number)
- REQUIRE(LogTestController::getInstance().contains(R"(</EventRecordID>)"));
-
REQUIRE(LogTestController::getInstance().contains(R"(<Channel>Application</Channel><Computer>)"));
- // the computer name goes here
-
REQUIRE(LogTestController::getInstance().contains(R"(</Computer><Security/></System><EventData><Data>Event
one</Data></EventData></Event>)"));
- }
+TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly",
"[onTrigger]") {
+ std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*",
"JSON", "Raw"}.run();
+ verifyJSON(event, R"json(
+ [
+ {
+ "name": "Event",
+ "children": [
+ {"name": "System"},
+ {
+ "name": "EventData",
+ "children": [{
+ "name": "Data",
+ "text": "Event one"
+ }]
+ }
+ ]
+ }
+ ]
+ )json");
}
namespace {
diff --git
a/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat
b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat
new file mode 100644
index 0000000..c74090b
--- /dev/null
+++
b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat
@@ -0,0 +1,62 @@
+@echo off &setlocal enabledelayedexpansion
+rem Licensed to the Apache Software Foundation (ASF) under one or more
+rem contributor license agreements. See the NOTICE file distributed with
+rem this work for additional information regarding copyright ownership.
+rem The ASF licenses this file to You under the Apache License, Version 2.0
+rem (the "License"); you may not use this file except in compliance with
+rem the License. You may obtain a copy of the License at
+rem
+rem http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+
+cd %1
+
+(
+ echo ^<?xml version="1.0" encoding="UTF-8"?^>
+ echo ^<instrumentationManifest
xsi:schemaLocation="http://schemas.microsoft.com/win/2004/08/events
eventman.xsd"
+ echo xmlns="http://schemas.microsoft.com/win/2004/08/events";
+ echo xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events";
+ echo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ echo xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ echo xmlns:trace="http://schemas.microsoft.com/win/2004/08/events/trace"^>
+ echo ^<instrumentation^>
+ echo ^<events^>
+ echo ^<provider name="minifi_unit_test_provider"
+ echo symbol="minifi_unit_test_provider"
+ echo guid="{ABCDEF01-8174-F1CA-87BE-DA129FF6001B}"
+ echo resourceFileName="%1\unit-test-provider.dll"
+ echo messageFileName="%1\unit-test-provider.dll"^>
+ echo ^<events^>
+ echo ^<event symbol="CustomEvent" value="10000" version="1"
channel="minifi_unit_test_provider/Log" template="CustomTemplate" /^>
+ echo ^</events^>
+ echo ^<levels/^>
+ echo ^<tasks/^>
+ echo ^<opcodes/^>
+ echo ^<channels^>
+ echo ^<channel name="minifi_unit_test_provider/Log" value="0x10"
type="Operational" enabled="true" /^>
+ echo ^</channels^>
+ echo ^<templates^>
+ echo ^<template tid="CustomTemplate"^>
+ echo ^<data name="param1" inType="win:UnicodeString"
outType="xs:string" /^>
+ echo ^<data name="param2" inType="win:UnicodeString"
outType="xs:string" /^>
+ echo ^<data name="Channel" inType="win:UnicodeString"
outType="xs:string" /^>
+ echo ^<binary /^>
+ echo ^</template^>
+ echo ^</templates^>
+ echo ^</provider^>
+ echo ^</events^>
+ echo ^</instrumentation^>
+ echo ^<localization/^>
+ echo ^</instrumentationManifest^>
+) > "%1/unit-test-provider.man"
+
+mc -css Namespace unit-test-provider.man
+mc -um unit-test-provider.man
+rc unit-test-provider.rc
+csc /target:library /unsafe /win32res:unit-test-provider.res
unit-test-provider.cs
+wevtutil im unit-test-provider.man
diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp
b/extensions/windows-event-log/wel/JSONUtils.cpp
new file mode 100644
index 0000000..4e59de7
--- /dev/null
+++ b/extensions/windows-event-log/wel/JSONUtils.cpp
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "JSONUtils.h"
+
+#include <vector>
+#include <string>
+#include <functional>
+
+#include <pugixml.hpp>
+
+#include "rapidjson/document.h"
+#include "rapidjson/writer.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/prettywriter.h"
+
+#include "gsl/gsl-lite.hpp";
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace wel {
+
+namespace {
+
+rapidjson::Value xmlElementToJSON(const pugi::xml_node& node,
rapidjson::Document& doc) {
+ gsl_Expects(node.type() == pugi::xml_node_type::node_element);
+ rapidjson::Value object(rapidjson::kObjectType);
+ object.AddMember("name", rapidjson::StringRef(node.name()),
doc.GetAllocator());
+ auto& attributes = object.AddMember("attributes", rapidjson::kObjectType,
doc.GetAllocator())["attributes"];
+ for (const auto& attr : node.attributes()) {
+ attributes.AddMember(rapidjson::StringRef(attr.name()),
rapidjson::StringRef(attr.value()), doc.GetAllocator());
+ }
+ auto& children = object.AddMember("children", rapidjson::kArrayType,
doc.GetAllocator())["children"];
+ for (const auto& child : node.children()) {
+ if (child.type() == pugi::xml_node_type::node_element) {
+ children.PushBack(xmlElementToJSON(child, doc), doc.GetAllocator());
+ }
+ }
+ object.AddMember("text", rapidjson::StringRef(node.text().get()),
doc.GetAllocator());
+ return object;
+}
+
+rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node,
rapidjson::Document& doc) {
+ gsl_Expects(node.type() == pugi::xml_node_type::node_document);
+ rapidjson::Value children(rapidjson::kArrayType);
+ for (const auto& child : node.children()) {
+ if (child.type() == pugi::xml_node_type::node_element) {
+ children.PushBack(xmlElementToJSON(child, doc), doc.GetAllocator());
+ }
+ }
+ return children;
+}
+
+rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
+ rapidjson::Document doc{rapidjson::kObjectType};
+
+ auto event_xml = root.child("Event");
+
+ {
+ auto system_xml = event_xml.child("System");
+ auto& system = flatten ? doc : doc.AddMember("System",
rapidjson::kObjectType, doc.GetAllocator())["System"];
+
+ {
+ auto provider_xml = system_xml.child("Provider");
+ auto& provider = flatten ? doc : system.AddMember("Provider",
rapidjson::kObjectType, doc.GetAllocator())["Provider"];
+ provider.AddMember("Name",
rapidjson::StringRef(provider_xml.attribute("Name").value()),
doc.GetAllocator());
+ provider.AddMember("Guid",
rapidjson::StringRef(provider_xml.attribute("Guid").value()),
doc.GetAllocator());
+ }
+
+ system.AddMember("EventID",
rapidjson::StringRef(system_xml.child("EventID").text().get()),
doc.GetAllocator());
+ system.AddMember("Version",
rapidjson::StringRef(system_xml.child("Version").text().get()),
doc.GetAllocator());
+ system.AddMember("Level",
rapidjson::StringRef(system_xml.child("Level").text().get()),
doc.GetAllocator());
+ system.AddMember("Task",
rapidjson::StringRef(system_xml.child("Task").text().get()),
doc.GetAllocator());
+ system.AddMember("Opcode",
rapidjson::StringRef(system_xml.child("Opcode").text().get()),
doc.GetAllocator());
+ system.AddMember("Keywords",
rapidjson::StringRef(system_xml.child("Keywords").text().get()),
doc.GetAllocator());
+
+ {
+ auto timeCreated_xml = system_xml.child("TimeCreated");
+ auto& timeCreated = flatten ? doc : system.AddMember("TimeCreated",
rapidjson::kObjectType, doc.GetAllocator())["TimeCreated"];
+ timeCreated.AddMember("SystemTime",
rapidjson::StringRef(timeCreated_xml.attribute("SystemTime").value()),
doc.GetAllocator());
+ }
+
+ system.AddMember("EventRecordID",
rapidjson::StringRef(system_xml.child("EventRecordID").text().get()),
doc.GetAllocator());
+
+ {
+ auto correlation_xml = system_xml.child("Correlation");
+ auto& correlation = flatten ? doc : system.AddMember("Correlation",
rapidjson::kObjectType, doc.GetAllocator())["Correlation"];
+ correlation.AddMember("ActivityID",
rapidjson::StringRef(correlation_xml.attribute("ActivityID").value()),
doc.GetAllocator());
+ }
+
+ {
+ auto execution_xml = system_xml.child("Execution");
+ auto& execution = flatten ? doc : system.AddMember("Execution",
rapidjson::kObjectType, doc.GetAllocator())["Execution"];
+ execution.AddMember("ProcessID",
rapidjson::StringRef(execution_xml.attribute("ProcessID").value()),
doc.GetAllocator());
+ execution.AddMember("ThreadID",
rapidjson::StringRef(execution_xml.attribute("ThreadID").value()),
doc.GetAllocator());
+ }
+
+ system.AddMember("Channel",
rapidjson::StringRef(system_xml.child("Channel").text().get()),
doc.GetAllocator());
+ system.AddMember("Computer",
rapidjson::StringRef(system_xml.child("Computer").text().get()),
doc.GetAllocator());
+ }
+
+ {
+ auto eventData_xml = event_xml.child("EventData");
+ // create EventData subarray even if flatten requested
+ doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
+ for (const auto& data : eventData_xml.children()) {
+ auto name_attr = data.attribute("Name");
+ rapidjson::Value item(rapidjson::kObjectType);
+ item.AddMember("Name", rapidjson::StringRef(name_attr.value()),
doc.GetAllocator());
+ item.AddMember("Content", rapidjson::StringRef(data.text().get()),
doc.GetAllocator());
+ item.AddMember("Type", rapidjson::StringRef(data.name()),
doc.GetAllocator());
+ // we need to query EventData because a reference to it wouldn't be
stable, as we
+ // possibly add members to its parent which could result in reallocation
+ doc["EventData"].PushBack(item, doc.GetAllocator());
+ // check collision
+ if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
+ doc.AddMember(rapidjson::StringRef(name_attr.value()),
rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+ }
+ }
+ }
+
+ return doc;
+}
+
+} // namespace
+
+rapidjson::Document toRawJSON(const pugi::xml_node& root) {
+ rapidjson::Document doc;
+ if (root.type() == pugi::xml_node_type::node_document) {
+ static_cast<rapidjson::Value&>(doc) = xmlDocumentToJSON(root, doc);
+ }
+ return doc;
+}
+
+rapidjson::Document toSimpleJSON(const pugi::xml_node& root) {
+ return toJSONImpl(root, false);
+}
+
+rapidjson::Document toFlattenedJSON(const pugi::xml_node& root) {
+ return toJSONImpl(root, true);
+}
+
+std::string jsonToString(rapidjson::Document& doc) {
+ rapidjson::StringBuffer buffer;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+ doc.Accept(writer);
+ return buffer.GetString();
+}
+
+} // namespace wel
+} // namespace minifi
+} // namespace nifi
+} // namespace apache
+} // namespace org
diff --git a/extensions/windows-event-log/wel/JSONUtils.h
b/extensions/windows-event-log/wel/JSONUtils.h
new file mode 100644
index 0000000..5b3b8ab
--- /dev/null
+++ b/extensions/windows-event-log/wel/JSONUtils.h
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#undef RAPIDJSON_ASSERT
+#define RAPIDJSON_ASSERT(x) if (!(x)) throw std::logic_error("rapidjson
exception"); // NOLINT
+
+#include <pugixml.hpp>
+
+#include <stdexcept> // for std::logic_error
+#include "rapidjson/document.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace wel {
+
+/**
+ * * !!WARNING!! The json document must not outlive the xml argument. For
better performance,
+ * the created json document stores references to values in the xml node.
Accessing the
+ * json document after the xml node has been changed or destroyed results in
undefined behavior.
+ *
+ * Converts each xml element node to a json object of
+ * the form {name: String, attributes: Object, children: Array, text: String}
+ * Aims to preserve most of the input xml structure.
+ */
+rapidjson::Document toRawJSON(const pugi::xml_node& root);
+
+/**
+ * * !!WARNING!! The json document must not outlive the xml argument. For
better performance,
+ * the created json document stores references to values in the xml node.
Accessing the
+ * json document after the xml node has been changed or destroyed results in
undefined behavior.
+ *
+ * Retains some hierarchical structure of the original xml event,
+ * e.g. transforms
+ * <Event><System><Provider Name="Banana" Guid="{5}"/></System></Event>
+ * into
+ * {System: {Provider: {Name: "Banana", Guid: "{5}"}}}
+ */
+rapidjson::Document toSimpleJSON(const pugi::xml_node& root);
+
+/**
+ * * !!WARNING!! The json document must not outlive the xml argument. For
better performance,
+ * the created json document stores references to values in the xml node.
Accessing the
+ * json document after the xml node has been changed or destroyed results in
undefined behavior.
+ *
+ * Flattens most of the structure, i.e. removes intermediate
+ * objects and lifts innermost string-valued keys to the root.
+ * e.g. {System: {Provider: {Name: String}}} => {Name: String}
+ *
+ * Moreover it also flattens each named data element where the
+ * name does not conflict with already existing members
+ * (e.g. a data with name "Guid" won't be flattened as it would
+ * overwrite the existing "Guid" field).
+ *
+ * e.g. {EventData: [{Name: "Test", Content: "X"}]} => {Test: "X"}
+ *
+ * In order to mitigate data loss, it preserves the EventData
+ * array in its entirety as well.
+ * (otherwise a "Guid" data would be lost)
+ */
+rapidjson::Document toFlattenedJSON(const pugi::xml_node& root);
+
+std::string jsonToString(rapidjson::Document& doc);
+
+} // namespace wel
+} // namespace minifi
+} // namespace nifi
+} // namespace apache
+} // namespace org
diff --git a/extensions/windows-event-log/wel/MetadataWalker.cpp
b/extensions/windows-event-log/wel/MetadataWalker.cpp
index 2c8e682..0f4f46f 100644
--- a/extensions/windows-event-log/wel/MetadataWalker.cpp
+++ b/extensions/windows-event-log/wel/MetadataWalker.cpp
@@ -79,7 +79,10 @@ bool MetadataWalker::for_each(pugi::xml_node &node) {
metadata_["EventID"] = node.text().get();
}
else {
- static std::map<std::string, EVT_FORMAT_MESSAGE_FLAGS> formatFlagMap = {
{"Channel", EvtFormatMessageChannel}, {"Keywords", EvtFormatMessageKeyword},
{"Level", EvtFormatMessageLevel}, {"Opcode", EvtFormatMessageOpcode},
{"Task",EvtFormatMessageTask} };
+ static std::map<std::string, EVT_FORMAT_MESSAGE_FLAGS> formatFlagMap = {
+ {"Channel", EvtFormatMessageChannel}, {"Keywords",
EvtFormatMessageKeyword}, {"Level", EvtFormatMessageLevel},
+ {"Opcode", EvtFormatMessageOpcode}, {"Task",EvtFormatMessageTask}
+ };
auto it = formatFlagMap.find(node_name);
if (it != formatFlagMap.end()) {
std::function<std::string(const std::string &)> updateFunc = [&](const
std::string &input) -> std::string {
diff --git a/extensions/windows-event-log/wel/WindowsEventLog.cpp
b/extensions/windows-event-log/wel/WindowsEventLog.cpp
index 8d1334b..80fb3c1 100644
--- a/extensions/windows-event-log/wel/WindowsEventLog.cpp
+++ b/extensions/windows-event-log/wel/WindowsEventLog.cpp
@@ -148,6 +148,11 @@ std::string
WindowsEventLogMetadataImpl::getEventData(EVT_FORMAT_MESSAGE_FLAGS f
EvtFormatMessage(metadata_ptr_, event_ptr_, 0, 0, NULL, flags,
num_chars_in_buffer, buffer.get(), &num_chars_used);
}
}
+
+ if (num_chars_used == 0) {
+ return event_data;
+ }
+
if (EvtFormatMessageKeyword == flags) {
buffer.get()[num_chars_used - 1] = L'\0';
}
diff --git a/win_build_vs.bat b/win_build_vs.bat
index 9335e08..222f2df 100755
--- a/win_build_vs.bat
+++ b/win_build_vs.bat
@@ -28,6 +28,7 @@ set build_coap=OFF
set build_jni=OFF
set build_SQL=OFF
set build_AWS=OFF
+set test_custom_wel_provider=OFF
set generator="Visual Studio 15 2017"
set cpack=OFF
set installer_merge_modules=OFF
@@ -50,14 +51,14 @@ for %%x in (%*) do (
if [%%~x] EQU [/64] set build_platform=x64
if [%%~x] EQU [/D] set cmake_build_type=RelWithDebInfo
if [%%~x] EQU [/DD] set cmake_build_type=Debug
- if [%%~x] EQU [/CI] set
"strict_gsl_checks=-DSTRICT_GSL_CHECKS=AUDIT"
+ if [%%~x] EQU [/CI] set
"strict_gsl_checks=-DSTRICT_GSL_CHECKS=AUDIT" & set test_custom_wel_provider=ON
if [%%~x] EQU [/NONFREEUCRT] set "redist=-DMSI_REDISTRIBUTE_UCRT_NONASL=ON"
)
mkdir %builddir%
pushd %builddir%\
-cmake -G %generator% -A %build_platform%
-DINSTALLER_MERGE_MODULES=%installer_merge_modules% -DENABLE_SQL=%build_SQL%
-DCMAKE_BUILD_TYPE_INIT=%cmake_build_type%
-DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32
-DENABLE_LIBRDKAFKA=%build_kafka% -DENABLE_JNI=%build_jni% -DOPENSSL_OFF=OFF
-DENABLE_COAP=%build_coap% -DENABLE_AWS=%build_AWS% -DUSE_SHARED_LIBS=OFF
-DDISABLE_CONTROLLER=ON -DBUILD_ROCKSDB=ON -DFORCE_WINDOWS=ON
-DUSE_SYSTEM_UUID=OFF -DDISABLE_LIBARCHIVE=OFF -DDISABLE_SCRIPTIN [...]
+cmake -G %generator% -A %build_platform%
-DINSTALLER_MERGE_MODULES=%installer_merge_modules%
-DTEST_CUSTOM_WEL_PROVIDER=%test_custom_wel_provider% -DENABLE_SQL=%build_SQL%
-DCMAKE_BUILD_TYPE_INIT=%cmake_build_type%
-DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32
-DENABLE_LIBRDKAFKA=%build_kafka% -DENABLE_JNI=%build_jni% -DOPENSSL_OFF=OFF
-DENABLE_COAP=%build_coap% -DENABLE_AWS=%build_AWS% -DUSE_SHARED_LIBS=OFF
-DDISABLE_CONTROLLER=ON -DBUILD_ROCKSDB=ON -DFORCE_WINDOWS=ON -DUSE_SYSTE [...]
IF %ERRORLEVEL% NEQ 0 EXIT /b %ERRORLEVEL%
if [%cpack%] EQU [ON] (
cpack -C %cmake_build_type%
