This is an automated email from the ASF dual-hosted git repository.
phrocker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/master by this push:
new a4cd78b WIP: Minificpp 1025 -- add metadata (#648)
a4cd78b is described below
commit a4cd78ba35e4c8ea10aa3c9046bc57646043f0c0
Author: Marc <[email protected]>
AuthorDate: Thu Sep 26 14:49:37 2019 -0400
WIP: Minificpp 1025 -- add metadata (#648)
* MINIFICPP-1025: Add message string functionality
* minor update
* updates
* MINIFICPP-1025: Update CWEL
* updates
* update message type
* update message type
* MINIFICPP-1025: add multiple SID resolution test
---
.../windows-event-log/ConsumeWindowsEventLog.cpp | 122 ++++++++++---
.../windows-event-log/ConsumeWindowsEventLog.h | 18 +-
.../tests/MetadataWalkerTests.cpp | 30 ++++
.../tests/resources/multiplesids.xml | 24 +++
.../windows-event-log/wel/MetadataWalker.cpp | 103 ++++++++++-
extensions/windows-event-log/wel/MetadataWalker.h | 36 ++--
.../windows-event-log/wel/WindowsEventLog.cpp | 200 +++++++++++++++++++++
extensions/windows-event-log/wel/WindowsEventLog.h | 193 ++++++++++++++++++++
libminifi/include/utils/Deleters.h | 40 +++++
9 files changed, 728 insertions(+), 38 deletions(-)
diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
index b2d6114..6e7b2e0 100644
--- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
+++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp
@@ -36,7 +36,9 @@
#include "core/ProcessContext.h"
#include "core/ProcessSession.h"
+
#pragma comment(lib, "wevtapi.lib")
+#pragma comment(lib, "ole32.lib")
namespace org {
namespace apache {
@@ -44,6 +46,8 @@ namespace nifi {
namespace minifi {
namespace processors {
+
+
static std::string to_string(const wchar_t* pChar) {
return std::wstring_convert<std::codecvt_utf8<wchar_t>>().to_bytes(pChar);
}
@@ -108,6 +112,21 @@ core::Property ConsumeWindowsEventLog::ResolveAsAttributes(
withDescription("If true, any metadata that is resolved ( such as IDs
or keyword metadata ) will be placed into attributes, otherwise it will be
replaced in the XML or text output")->
build());
+
+core::Property ConsumeWindowsEventLog::EventHeaderDelimiter(
+ core::PropertyBuilder::createProperty("Event Header Delimiter")->
+ isRequired(false)->
+ withDescription("If set, the chosen delimiter will be used in the Event
output header. Otherwise, a colon followed by spaces will be used.")->
+ build());
+
+
+core::Property ConsumeWindowsEventLog::EventHeader(
+ core::PropertyBuilder::createProperty("Event Header")->
+ isRequired(false)->
+ withDefaultValue("LOG_NAME=Log Name, SOURCE = Source, TIME_CREATED =
Date,EVENT_RECORDID=Record ID,EVENTID = Event ID,TASK_CATEGORY = Task
Category,LEVEL = Level,KEYWORDS = Keywords,USER = User,COMPUTER = Computer,
EVENT_TYPE = EventType")->
+ withDescription("Comma seperated list of key/value pairs with the
following keys LOG_NAME, SOURCE,
TIME_CREATED,EVENT_RECORDID,EVENTID,TASK_CATEGORY,LEVEL,KEYWORDS,USER,COMPUTER,
and EVENT_TYPE. Eliminating fields will remove them from the header.")->
+ build());
+
core::Relationship ConsumeWindowsEventLog::Success("success", "Relationship
for successfully consumed events.");
ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name,
utils::Identifier uuid)
@@ -127,16 +146,53 @@ ConsumeWindowsEventLog::~ConsumeWindowsEventLog() {
void ConsumeWindowsEventLog::initialize() {
//! Set the supported properties
- setSupportedProperties({Channel, Query, MaxBufferSize,
InactiveDurationToReconnect, IdentifierMatcher, IdentifierFunction,
ResolveAsAttributes });
+ setSupportedProperties({Channel, Query, MaxBufferSize,
InactiveDurationToReconnect, IdentifierMatcher, IdentifierFunction,
ResolveAsAttributes, EventHeaderDelimiter, EventHeader });
//! Set the supported relationships
setSupportedRelationships({Success});
}
+bool ConsumeWindowsEventLog::insertHeaderName(wel::METADATA_NAMES &header,
const std::string &key, const std::string & value) {
+
+ wel::METADATA name =
wel::WindowsEventLogMetadata::getMetadataFromString(key);
+
+ if (name != wel::METADATA::UNKNOWN) {
+ header.emplace_back(std::make_pair(name, value));
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
void ConsumeWindowsEventLog::onSchedule(const
std::shared_ptr<core::ProcessContext> &context, const
std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) {
context->getProperty(IdentifierMatcher.getName(), regex_);
context->getProperty(ResolveAsAttributes.getName(),
resolve_as_attributes_);
context->getProperty(IdentifierFunction.getName(),
apply_identifier_function_);
+ context->getProperty(EventHeaderDelimiter.getName(), header_delimiter_);
+
+ std::string header;
+ context->getProperty(EventHeader.getName(), header);
+
+ auto keyValueSplit = utils::StringUtils::split(header, ",");
+ for (const auto &kv : keyValueSplit) {
+ auto splitKeyAndValue = utils::StringUtils::split(kv, "=");
+ if (splitKeyAndValue.size() == 2) {
+ auto key =
utils::StringUtils::trim(splitKeyAndValue.at(0));
+ auto value =
utils::StringUtils::trim(splitKeyAndValue.at(1));
+ if (!insertHeaderName(header_names_, key, value)) {
+ logger_->log_debug("%s is an invalid key for
the header map", key);
+ }
+ }
+ else if (splitKeyAndValue.size() == 1) {
+ auto key =
utils::StringUtils::trim(splitKeyAndValue.at(0));
+ if (!insertHeaderName(header_names_, key, "")) {
+ logger_->log_debug("%s is an invalid key for
the header map", key);
+ }
+ }
+
+ }
+
if (subscriptionHandle_) {
logger_->log_error("Processor already subscribed to Event Log, expected
cleanup to unsubscribe.");
} else {
@@ -144,6 +200,7 @@ void ConsumeWindowsEventLog::onSchedule(const
std::shared_ptr<core::ProcessConte
subscribe(context);
}
+
}
void ConsumeWindowsEventLog::onTrigger(const
std::shared_ptr<core::ProcessContext> &context, const
std::shared_ptr<core::ProcessSession> &session) {
@@ -169,7 +226,7 @@ void ConsumeWindowsEventLog::onTrigger(const
std::shared_ptr<core::ProcessContex
}
}
-EVT_HANDLE ConsumeWindowsEventLog::getProvider(const std::string & name) {
+wel::WindowsEventLogHandler ConsumeWindowsEventLog::getEventLogHandler(const
std::string & name) {
std::lock_guard<std::mutex> lock(cache_mutex_);
auto provider = providers_.find(name);
if (provider != std::end(providers_)) {
@@ -179,14 +236,14 @@ EVT_HANDLE ConsumeWindowsEventLog::getProvider(const
std::string & name) {
std::wstring temp_wstring = std::wstring(name.begin(), name.end());
LPCWSTR widechar = temp_wstring.c_str();
- providers_[name] = EvtOpenPublisherMetadata(NULL, widechar, NULL, 0, 0);
+ providers_[name] =
wel::WindowsEventLogHandler(EvtOpenPublisherMetadata(NULL, widechar, NULL, 0,
0));
return providers_[name];
}
+
bool ConsumeWindowsEventLog::subscribe(const
std::shared_ptr<core::ProcessContext> &context) {
- std::string channel;
- context->getProperty(Channel.getName(), channel);
+ context->getProperty(Channel.getName(), channel_);
std::string query;
context->getProperty(Query.getName(), query);
@@ -194,7 +251,7 @@ bool ConsumeWindowsEventLog::subscribe(const
std::shared_ptr<core::ProcessContex
context->getProperty(MaxBufferSize.getName(), maxBufferSize_);
logger_->log_debug("ConsumeWindowsEventLog: maxBufferSize_ %lld",
maxBufferSize_);
- provenanceUri_ = "winlog://" + computerName_ + "/" + channel + "?" + query;
+ provenanceUri_ = "winlog://" + computerName_ + "/" + channel_ + "?" + query;
std::string strInactiveDurationToReconnect;
context->getProperty(InactiveDurationToReconnect.getName(),
strInactiveDurationToReconnect);
@@ -209,37 +266,37 @@ bool ConsumeWindowsEventLog::subscribe(const
std::shared_ptr<core::ProcessContex
subscriptionHandle_ = EvtSubscribe(
NULL,
NULL,
- std::wstring(channel.begin(), channel.end()).c_str(),
+ std::wstring(channel_.begin(), channel_.end()).c_str(),
std::wstring(query.begin(), query.end()).c_str(),
NULL,
this,
- [](EVT_SUBSCRIBE_NOTIFY_ACTION action, PVOID pContext, EVT_HANDLE hEvent)
+ [](EVT_SUBSCRIBE_NOTIFY_ACTION action, PVOID pContext, EVT_HANDLE
eventHandle)
{
+
auto pConsumeWindowsEventLog =
static_cast<ConsumeWindowsEventLog*>(pContext);
auto& logger = pConsumeWindowsEventLog->logger_;
if (action == EvtSubscribeActionError) {
- if (ERROR_EVT_QUERY_RESULT_STALE == (DWORD)hEvent) {
+ if (ERROR_EVT_QUERY_RESULT_STALE == (DWORD)eventHandle) {
logger->log_error("Received missing event notification. Consider
triggering processor more frequently or increasing queue size.");
} else {
- logger->log_error("Received the following Win32 error: %x",
hEvent);
+ logger->log_error("Received the following Win32 error: %x",
eventHandle);
}
} else if (action == EvtSubscribeActionDeliver) {
DWORD size = 0;
DWORD used = 0;
DWORD propertyCount = 0;
-
- if (!EvtRender(NULL, hEvent, EvtRenderEventXml, size, 0, &used,
&propertyCount)) {
+ if (!EvtRender(NULL, eventHandle, EvtRenderEventXml, size, 0, &used,
&propertyCount)) {
if (ERROR_INSUFFICIENT_BUFFER == GetLastError()) {
if (used > pConsumeWindowsEventLog->maxBufferSize_) {
- logger->log_error("Dropping event %x because it couldn't be
rendered within %ll bytes.", hEvent, pConsumeWindowsEventLog->maxBufferSize_);
+ logger->log_error("Dropping event %x because it couldn't be
rendered within %ll bytes.", eventHandle,
pConsumeWindowsEventLog->maxBufferSize_);
return 0UL;
}
size = used;
std::vector<wchar_t> buf(size/2 + 1);
- if (EvtRender(NULL, hEvent, EvtRenderEventXml, size, &buf[0],
&used, &propertyCount)) {
+ if (EvtRender(NULL, eventHandle, EvtRenderEventXml, size,
&buf[0], &used, &propertyCount)) {
std::string xml = to_string(&buf[0]);
EventRender renderedData;
@@ -247,19 +304,37 @@ bool ConsumeWindowsEventLog::subscribe(const
std::shared_ptr<core::ProcessContex
pugi::xml_document doc;
pugi::xml_parse_result result =
doc.load_string(xml.c_str());
-
-
if (!result) {
- logger->log_error("'loadXML' failed");
+ logger->log_error("Invalid XML
produced");
return 0UL;
}
+ // this is a well known path.
+ std::string providerName =
doc.child("Event").child("System").child("Provider").attribute("Name").value();
+
+ auto handler =
pConsumeWindowsEventLog->getEventLogHandler(providerName);
+ auto message =
handler.getEventMessage(eventHandle);
+
- std::string providerName =
doc.child("System").child("Provider").attribute("Name").value();
// resolve the event metadata
- wel::MetadataWalker
walker(pConsumeWindowsEventLog->getProvider(providerName), hEvent,
!pConsumeWindowsEventLog->resolve_as_attributes_,
pConsumeWindowsEventLog->apply_identifier_function_,
pConsumeWindowsEventLog->regex_);
+ wel::MetadataWalker
walker(pConsumeWindowsEventLog->getEventLogHandler(providerName).getMetadata(),
pConsumeWindowsEventLog->channel_, eventHandle,
!pConsumeWindowsEventLog->resolve_as_attributes_,
pConsumeWindowsEventLog->apply_identifier_function_,
pConsumeWindowsEventLog->regex_);
doc.traverse(walker);
+ if (!message.empty())
+ {
+ for (const auto &mapEntry :
walker.getIdentifiers()) {
+ // replace the identifiers with
their translated strings.
+
utils::StringUtils::replaceAll(message, mapEntry.first, mapEntry.second);
+ }
+ wel::WindowsEventLogHeader
log_header(pConsumeWindowsEventLog->header_names_);
+ // set the delimiter
+
log_header.setDelimiter(pConsumeWindowsEventLog->header_delimiter_);
+ // render the header.
+ renderedData.rendered_text_ =
log_header.getEventHeader(&walker);
+ renderedData.rendered_text_ +=
"Message" + pConsumeWindowsEventLog->header_delimiter_ + " ";
+ renderedData.rendered_text_ += message;
+ }
+
if
(pConsumeWindowsEventLog->resolve_as_attributes_) {
renderedData.matched_fields_ =
walker.getFieldValues();
}
@@ -329,7 +404,14 @@ int ConsumeWindowsEventLog::processQueue(const
std::shared_ptr<core::ProcessSess
session->putAttribute(flowFile, FlowAttributeKey(MIME_TYPE),
"application/xml");
session->getProvenanceReporter()->receive(flowFile, provenanceUri_,
getUUIDStr(), "Consume windows event logs", 0);
session->transfer(flowFile, Success);
- session->commit();
+
+ flowFile = session->create();
+
+ session->write(flowFile, &WriteCallback(evt.rendered_text_));
+ session->putAttribute(flowFile, FlowAttributeKey(MIME_TYPE),
"text/plain");
+ session->getProvenanceReporter()->receive(flowFile, provenanceUri_,
getUUIDStr(), "Consume windows event logs", 0);
+ session->transfer(flowFile, Success);
+ session->commit();
flowFileCount++;
}
diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h
b/extensions/windows-event-log/ConsumeWindowsEventLog.h
index 4d6196d..dae262c 100644
--- a/extensions/windows-event-log/ConsumeWindowsEventLog.h
+++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h
@@ -21,6 +21,7 @@
#pragma once
#include "core/Core.h"
+#include "wel/WindowsEventLog.h"
#include "FlowFileRecord.h"
#include "concurrentqueue.h"
#include "core/Processor.h"
@@ -31,6 +32,7 @@
#include <regex>
#include <codecvt>
#include "utils/OsUtils.h"
+#include <Objbase.h>
//#import <msxml6.dll>
@@ -44,10 +46,9 @@ namespace processors {
struct EventRender {
std::map<std::string, std::string> matched_fields_;
std::string text_;
+ std::string rendered_text_;
};
-
-
//! ConsumeWindowsEventLog Class
class ConsumeWindowsEventLog : public core::Processor
{
@@ -73,6 +74,8 @@ public:
static core::Property IdentifierMatcher;
static core::Property IdentifierFunction;
static core::Property ResolveAsAttributes;
+ static core::Property EventHeaderDelimiter;
+ static core::Property EventHeader;
//! Supported Relationships
static core::Relationship Success;
@@ -90,17 +93,24 @@ public:
//! Initialize, overwrite by NiFi ConsumeWindowsEventLog
virtual void initialize(void) override;
virtual void notifyStop() override;
+
protected:
bool subscribe(const std::shared_ptr<core::ProcessContext> &context);
void unsubscribe();
int processQueue(const std::shared_ptr<core::ProcessSession> &session);
- EVT_HANDLE getProvider(const std::string & name);
+ wel::WindowsEventLogHandler getEventLogHandler(const std::string & name);
+
+ bool insertHeaderName(wel::METADATA_NAMES &header, const std::string &key,
const std::string &value);
void LogWindowsError();
private:
+
// Logger
+ wel::METADATA_NAMES header_names_;
+ std::string header_delimiter_;
+ std::string channel_;
std::shared_ptr<logging::Logger> logger_;
std::string regex_;
bool resolve_as_attributes_;
@@ -114,7 +124,7 @@ private:
DWORD lastActivityTimestamp_{};
std::shared_ptr<core::ProcessSessionFactory> sessionFactory_;
std::mutex cache_mutex_;
- std::map<std::string, EVT_HANDLE > providers_;
+ std::map<std::string, wel::WindowsEventLogHandler > providers_;
};
REGISTER_RESOURCE(ConsumeWindowsEventLog, "Windows Event Log Subscribe
Callback to receive FlowFiles from Events on Windows.");
diff --git a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp
b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp
index aae70bf..b9531b0 100644
--- a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp
+++ b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp
@@ -93,4 +93,34 @@ TEST_CASE("TestUnknownSid", "[InvalidSet]") {
REQUIRE(MetadataWalker::updateXmlMetadata(xml, 0x00, 0x00, false, true)
== formatXml(xml));
+}
+
+
+
+TEST_CASE("TestMultipleSids", "[Resolutions]") {
+ std::ifstream unresolvedfile("resources/multiplesids.xml");
+ std::string xml((std::istreambuf_iterator<char>(unresolvedfile)),
+ std::istreambuf_iterator<char>());
+
+ std::string programmaticallyResolved;
+
+ pugi::xml_document doc;
+ xml = MetadataWalker::updateXmlMetadata(xml, 0x00, 0x00, false, true);
+ pugi::xml_parse_result result = doc.load_string(xml.c_str());
+
+ for (const auto &node :
doc.child("Event").child("EventData").children())
+ {
+ auto name = node.attribute("Name").as_string();
+ if
(utils::StringUtils::equalsIgnoreCase("GroupMembership",name)) {
+ programmaticallyResolved = node.text().get();
+ break;
+ }
+ }
+
+ std::string expected = "Nobody Everyone Null Authority";
+
+ // we are only testing mulitiple sid resolutions, not the resolution of
other items.
+ REQUIRE(expected == programmaticallyResolved);
+
+
}
\ No newline at end of file
diff --git a/extensions/windows-event-log/tests/resources/multiplesids.xml
b/extensions/windows-event-log/tests/resources/multiplesids.xml
new file mode 100644
index 0000000..b5c35b5
--- /dev/null
+++ b/extensions/windows-event-log/tests/resources/multiplesids.xml
@@ -0,0 +1,24 @@
+
+ <Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event";>
+ <System>
+ <Provider Name="Microsoft-Windows-Security-Auditing"
Guid="{54849625-5478-4994-a5ba-3e3b0328c30d}" />
+ <EventID>4672</EventID>
+ <Version>0</Version>
+ <Level>0</Level>
+ <Task>12548</Task>
+ <Opcode>0</Opcode>
+ <Keywords>0x8020000000000000</Keywords>
+ <TimeCreated SystemTime="2019-08-27T14:37:56.104234800Z" />
+ <EventRecordID>2575952</EventRecordID>
+ <Correlation ActivityID="{47aa1c15-3cc3-0006-e859-7fa1025cd501}" />
+ <Execution ProcessID="840" ThreadID="11544" />
+ <Channel>Security</Channel>
+ <Computer>TestComputer</Computer>
+ <Security />
+ </System>
+ <EventData>
+ <Data Name="SubjectUserSid">S-1-0-0</Data>
+ <Data Name="PrivilegeList">SeAssignPrimaryTokenPrivilege SeTcbPrivilege
SeSecurityPrivilege SeTakeOwnershipPrivilege SeLoadDriverPrivilege
SeBackupPrivilege SeRestorePrivilege SeDebugPrivilege SeAuditPrivilege
SeSystemEnvironmentPrivilege SeImpersonatePrivilege
SeDelegateSessionUserImpersonatePrivilege</Data>
+ <Data Name="GroupMembership">%{S-1-0-0} %{S-1-1-0} %{S-1-0}</Data>
+ </EventData>
+ </Event>
\ No newline at end of file
diff --git a/extensions/windows-event-log/wel/MetadataWalker.cpp
b/extensions/windows-event-log/wel/MetadataWalker.cpp
index af57788..3c10006 100644
--- a/extensions/windows-event-log/wel/MetadataWalker.cpp
+++ b/extensions/windows-event-log/wel/MetadataWalker.cpp
@@ -1,5 +1,7 @@
+#include <windows.h>
#include "MetadataWalker.h"
#include "XMLString.h"
+#include <strsafe.h>
namespace org {
namespace apache {
@@ -17,21 +19,62 @@ bool MetadataWalker::for_each(pugi::xml_node &node) {
if (std::regex_match(attr.name(), regex_)) {
std::function<std::string(const std::string &)>
idUpdate = [&](const std::string &input) -> std::string {
- return resolve_ ?
utils::OsUtils::userIdToUsername(input) : input;
+ if (resolve_) {
+ auto resolved =
utils::OsUtils::userIdToUsername(input);
+ replaced_identifiers_[input] =
resolved;
+ return resolved;
+ }
+ else {
+ replaced_identifiers_[input] =
input;
+ }
};
updateText(node, attr.name(),
std::move(idUpdate));
}
+
if (std::regex_match(attr.value(), regex_)) {
std::function<std::string(const std::string &)>
idUpdate = [&](const std::string &input) -> std::string {
- return resolve_ ?
utils::OsUtils::userIdToUsername(input) : input;
+ if (resolve_) {
+ auto resolved =
utils::OsUtils::userIdToUsername(input);
+ replaced_identifiers_[input] =
resolved;
+ return resolved;
+ }
+ else {
+ replaced_identifiers_[input] =
input;
+ }
};
updateText(node, attr.value(),
std::move(idUpdate));
}
}
+
+ if (resolve_) {
+ std::string nodeText = node.text().get();
+ std::vector<std::string> ids = getIdentifiers(nodeText);
+ for (const auto &id : ids) {
+ auto resolved =
utils::OsUtils::userIdToUsername(id);
+ std::string replacement = "%{" + id + "}";
+ replaced_identifiers_[id] = resolved;
+ replaced_identifiers_[replacement] = resolved;
+ nodeText =
utils::StringUtils::replaceAll(nodeText, replacement, resolved);
+ }
+ node.text().set(nodeText.c_str());
+ }
+
+ }
+ else if (node_name == "TimeCreated") {
+ metadata_["TimeCreated"] = node.attribute("SystemTime").value();
+ }
+ else if (node_name == "EventRecordID") {
+ metadata_["EventRecordID"] = node.text().get();
+ }
+ else if (node_name == "Provider") {
+ metadata_["Provider"] = node.attribute("Name").value();
+ }
+ else if (node_name == "EventID") {
+ metadata_["EventID"] = node.text().get();
}
else {
- static std::map<std::string, EVT_FORMAT_MESSAGE_FLAGS>
formatFlagMap = { {"Channel", EvtFormatMessageChannel}, {"Keywords",
EvtFormatMessageKeyword}, {"Level", EvtFormatMessageLevel}, {"Opcode",
EvtFormatMessageOpcode} };
+ static std::map<std::string, EVT_FORMAT_MESSAGE_FLAGS>
formatFlagMap = { {"Channel", EvtFormatMessageChannel}, {"Keywords",
EvtFormatMessageKeyword}, {"Level", EvtFormatMessageLevel}, {"Opcode",
EvtFormatMessageOpcode}, {"Task",EvtFormatMessageTask} };
auto it = formatFlagMap.find(node_name);
if (it != formatFlagMap.end()) {
std::function<std::string(const std::string &)>
updateFunc = [&](const std::string &input) -> std::string {
@@ -53,13 +96,64 @@ bool MetadataWalker::for_each(pugi::xml_node &node) {
return true;
}
+std::vector<std::string> MetadataWalker::getIdentifiers(const std::string
&text) const {
+ auto pos = text.find("%{");
+ std::vector<std::string> found_strings;
+ while (pos != std::string::npos)
+ {
+ auto next_pos = text.find("}", pos);
+ if (next_pos != std::string::npos) {
+ auto potential_identifier = text.substr(pos + 2,
next_pos - (pos + 2));
+ std::smatch match;
+ if (potential_identifier.find("S-") !=
std::string::npos){
+ found_strings.push_back(potential_identifier);
+ }
+ }
+ pos = text.find("%{", pos + 2);
+ }
+ return found_strings;
+}
+
+std::string MetadataWalker::getMetadata(METADATA metadata) const {
+ switch (metadata) {
+ case LOG_NAME:
+ return log_name_;
+ case SOURCE:
+ return getString(metadata_,"Provider");
+ case TIME_CREATED:
+ return event_timestamp_str_;
+ case EVENTID:
+ return getString(metadata_,"EventID");
+ case EVENT_RECORDID:
+ return getString(metadata_, "EventRecordID");
+ case OPCODE:
+ return getString(metadata_, "Opcode");
+ case TASK_CATEGORY:
+ return getString(metadata_,"Task");
+ case LEVEL:
+ return getString(metadata_,"Level");
+ case KEYWORDS:
+ return getString(metadata_,"Keywords");
+ case EVENT_TYPE:
+ return std::to_string(event_type_index_);
+ case COMPUTER:
+ return getComputerName();
+ };
+ return "N/A";
+}
+
std::map<std::string, std::string> MetadataWalker::getFieldValues() const {
return fields_values_;
}
+std::map<std::string, std::string> MetadataWalker::getIdentifiers() const {
+ return replaced_identifiers_;
+}
+
+
std::string MetadataWalker::updateXmlMetadata(const std::string &xml,
EVT_HANDLE metadata_ptr, EVT_HANDLE event_ptr, bool update_xml, bool resolve,
const std::string ®ex) {
- MetadataWalker walker(metadata_ptr, event_ptr, update_xml, resolve,
regex);
+ MetadataWalker walker(metadata_ptr,"", event_ptr, update_xml, resolve,
regex);
pugi::xml_document doc;
pugi::xml_parse_result result = doc.load_string(xml.c_str());
@@ -84,6 +178,7 @@ void MetadataWalker::updateText(pugi::xml_node &node, const
std::string &field_n
auto new_field_value = fn(previous_value);
if (new_field_value != previous_value) {
+ metadata_[field_name] = new_field_value;
if (update_xml_) {
node.text().set(new_field_value.c_str());
} else {
diff --git a/extensions/windows-event-log/wel/MetadataWalker.h
b/extensions/windows-event-log/wel/MetadataWalker.h
index 984103a..cf491d7 100644
--- a/extensions/windows-event-log/wel/MetadataWalker.h
+++ b/extensions/windows-event-log/wel/MetadataWalker.h
@@ -20,6 +20,7 @@
#pragma once
+#include "WindowsEventLog.h"
#include "core/Core.h"
#include "FlowFileRecord.h"
#include "concurrentqueue.h"
@@ -38,15 +39,16 @@ namespace nifi {
namespace minifi {
namespace wel {
+
+
/**
* Defines a tree walker for the XML input
*
*/
-class MetadataWalker : public pugi::xml_tree_walker {
+class MetadataWalker : public pugi::xml_tree_walker, public
WindowsEventLogMetadata {
public:
- MetadataWalker(EVT_HANDLE metadata_ptr, EVT_HANDLE event_ptr, bool
update_xml, bool resolve, const std::string ®ex = "")
- : metadata_ptr_(metadata_ptr),
- event_ptr_(event_ptr),
+ MetadataWalker(EVT_HANDLE metadata_ptr, const std::string &log_name,
EVT_HANDLE event_ptr, bool update_xml, bool resolve, const std::string ®ex =
"")
+ : WindowsEventLogMetadata(metadata_ptr, event_ptr, log_name),
regex_(regex),
regex_str_(regex),
update_xml_(update_xml),
@@ -61,13 +63,27 @@ class MetadataWalker : public pugi::xml_tree_walker {
static std::string updateXmlMetadata(const std::string &xml, EVT_HANDLE
metadata_ptr, EVT_HANDLE event_ptr, bool update_xml, bool resolve, const
std::string ®ex = "");
- std::map<std::string, std::string> getFieldValues() const;
+ virtual std::map<std::string, std::string> getFieldValues() const
override;
+
+ virtual std::map<std::string, std::string> getIdentifiers() const
override;
+
+ virtual std::string getMetadata(METADATA metadata) const override;
+
private:
- static std::string to_string(const wchar_t* pChar);
+ std::vector<std::string> getIdentifiers(const std::string &text) const
;
- /**
+
+ static std::string getString(const std::map<std::string, std::string>
&map, const std::string &field) {
+ auto srch = map.find(field);
+ if (srch != std::end(map)) {
+ return srch->second;
+ }
+ return "N/A";
+ }
+ static std::string to_string(const wchar_t* pChar);
+ /**
* Updates text within the XML representation
*/
void updateText(pugi::xml_node &node, const std::string &field_name,
std::function<std::string(const std::string &)> &&fn);
@@ -76,14 +92,14 @@ class MetadataWalker : public pugi::xml_tree_walker {
* Gets event data.
*/
std::string getEventData(EVT_FORMAT_MESSAGE_FLAGS flags);
-
- EVT_HANDLE metadata_ptr_;
- EVT_HANDLE event_ptr_;
+
std::regex regex_;
std::string regex_str_;
bool update_xml_;
bool resolve_;
+ std::map<std::string, std::string> metadata_;
std::map<std::string, std::string> fields_values_;
+ std::map<std::string, std::string> replaced_identifiers_;
};
diff --git a/extensions/windows-event-log/wel/WindowsEventLog.cpp
b/extensions/windows-event-log/wel/WindowsEventLog.cpp
new file mode 100644
index 0000000..4b2b041
--- /dev/null
+++ b/extensions/windows-event-log/wel/WindowsEventLog.cpp
@@ -0,0 +1,200 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <winmeta.h>
+#include "WindowsEventLog.h"
+#include "utils/Deleters.h"
+#include <algorithm>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace wel {
+
+
+void WindowsEventLogMetadata::renderMetadata() {
+ DWORD status = ERROR_SUCCESS;
+ DWORD dwBufferSize = 0;
+ DWORD dwBufferUsed = 0;
+ DWORD dwPropertyCount = 0;
+ std::unique_ptr< EVT_VARIANT, utils::FreeDeleter> rendered_values;
+
+ auto context = EvtCreateRenderContext(0, NULL, EvtRenderContextSystem);
+ if (!EvtRender(context, event_ptr_, EvtRenderEventValues, dwBufferSize,
nullptr, &dwBufferUsed, &dwPropertyCount))
+ {
+ if (ERROR_INSUFFICIENT_BUFFER == (status = GetLastError()))
+ {
+ dwBufferSize = dwBufferUsed;
+ rendered_values = std::unique_ptr<EVT_VARIANT,
utils::FreeDeleter>((PEVT_VARIANT)(malloc(dwBufferSize)));
+ if (rendered_values)
+ {
+ EvtRender(context, event_ptr_,
EvtRenderEventValues, dwBufferSize, rendered_values.get(), &dwBufferUsed,
&dwPropertyCount);
+ }
+ }
+ else {
+ return;
+ }
+
+ if (ERROR_SUCCESS != (status = GetLastError()))
+ {
+ return;
+ }
+ }
+
+ event_timestamp_ = static_cast<PEVT_VARIANT>(
rendered_values.get())[EvtSystemTimeCreated].FileTimeVal;
+
+ SYSTEMTIME st;
+ FILETIME ft;
+
+ ft.dwHighDateTime = (DWORD)((event_timestamp_ >> 32) & 0xFFFFFFFF);
+ ft.dwLowDateTime = (DWORD)(event_timestamp_ & 0xFFFFFFFF);
+
+ FileTimeToSystemTime(&ft, &st);
+ std::stringstream datestr;
+
+ std::string period = "AM";
+ auto hour = st.wHour;
+ if (hour >= 12 && hour < 24)
+ period = "PM";
+ if (hour >= 12)
+ hour -= 12;
+ datestr << st.wMonth << "/" << st.wDay << "/" << st.wYear << " " <<
std::setfill('0') << std::setw(2) << hour << ":" << std::setfill('0') <<
std::setw(2) << st.wMinute << ":" << std::setfill('0') << std::setw(2) <<
st.wSecond << " " << period;
+ event_timestamp_str_ = datestr.str();
+ auto level =
static_cast<PEVT_VARIANT>(rendered_values.get())[EvtSystemLevel];
+ auto keyword =
static_cast<PEVT_VARIANT>(rendered_values.get())[EvtSystemKeywords];
+ if (level.Type == EvtVarTypeByte) {
+ switch (level.ByteVal)
+ {
+ case WINEVENT_LEVEL_CRITICAL:
+ case WINEVENT_LEVEL_ERROR:
+ event_type_ = "Error";
+ event_type_index_ = 1;
+ break;
+ case WINEVENT_LEVEL_WARNING:
+ event_type_ = "Warning";
+ event_type_index_ = 2;
+ break;
+ case WINEVENT_LEVEL_INFO:
+ case WINEVENT_LEVEL_VERBOSE:
+ event_type_ = "Information";
+ event_type_index_ = 4;
+ break;
+ default:
+ event_type_index_ = 0;
+ };
+
+ }
+ else {
+ event_type_ = "N/A";
+ }
+
+ if (keyword.UInt64Val & WINEVENT_KEYWORD_AUDIT_SUCCESS) {
+ event_type_ = "Success Audit";
+ event_type_index_ = 8;
+ } else if (keyword.UInt64Val & EVENTLOG_AUDIT_FAILURE) {
+ event_type_ = "Failure Audit";
+ event_type_index_ = 16;
+ }
+
+
+
+}
+
+std::string WindowsEventLogHandler::getEventMessage(EVT_HANDLE eventHandle)
const
+{
+ std::string returnValue;
+ std::unique_ptr<WCHAR, utils::FreeDeleter> pBuffer;
+ DWORD dwBufferSize = 0;
+ DWORD dwBufferUsed = 0;
+ DWORD status = 0;
+
+ EvtFormatMessage(metadata_provider_, eventHandle, 0, 0, NULL,
EvtFormatMessageEvent, dwBufferSize, pBuffer.get(), &dwBufferUsed);
+
+ // we need to get the size of the buffer
+ status = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER == status) {
+ dwBufferSize = dwBufferUsed;
+
+ /* All C++ examples use malloc and even HeapAlloc in some
cases. To avoid any problems ( with EvtFormatMessage calling
+ free for example ) we will continue to use malloc and
use a custom deleter with unique_ptr.
+ '*/
+ pBuffer = std::unique_ptr<WCHAR,
utils::FreeDeleter>((LPWSTR)malloc(dwBufferSize * sizeof(WCHAR)));
+
+
+ if (pBuffer) {
+ EvtFormatMessage(metadata_provider_, eventHandle, 0, 0,
NULL, EvtFormatMessageEvent, dwBufferSize, pBuffer.get(), &dwBufferUsed);
+ }
+ else {
+ return returnValue;
+ }
+ }
+ else if (ERROR_EVT_MESSAGE_NOT_FOUND == status ||
ERROR_EVT_MESSAGE_ID_NOT_FOUND == status) {
+ return returnValue;
+ }
+ else {
+ return returnValue;
+ }
+
+ // convert wstring to std::string
+ std::wstring message(pBuffer.get());
+ returnValue = std::string(message.begin(), message.end());
+ return returnValue;
+
+}
+
+void WindowsEventLogHeader::setDelimiter(const std::string &delim) {
+ delimiter_ = delim;
+}
+
+std::string WindowsEventLogHeader::createDefaultDelimiter(size_t max, size_t
length) const {
+ if (max > length) {
+ return ":" + std::string(max - length, ' ');
+ }
+ else {
+ return ": ";
+ }
+}
+
+std::string WindowsEventLogHeader::getEventHeader(const
WindowsEventLogMetadata * const metadata) const{
+ std::stringstream eventHeader;
+ size_t max = 1;
+ for (const auto &option : header_names_) {
+ max = (std::max(max, option.second.size()));
+ }
+ ++max; // increment by one to get space.
+ for (const auto &option : header_names_) {
+ auto name = option.second;
+ if (!name.empty()) {
+ eventHeader << name << (delimiter_.empty() ?
createDefaultDelimiter(max, name.size()) : delimiter_);
+ }
+ eventHeader <<
utils::StringUtils::trim(metadata->getMetadata(option.first)) << std::endl;
+ }
+
+ return eventHeader.str();
+}
+
+EVT_HANDLE WindowsEventLogHandler::getMetadata() const {
+ return metadata_provider_;
+}
+
+} /* namespace wel */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
diff --git a/extensions/windows-event-log/wel/WindowsEventLog.h
b/extensions/windows-event-log/wel/WindowsEventLog.h
new file mode 100644
index 0000000..7f8004b
--- /dev/null
+++ b/extensions/windows-event-log/wel/WindowsEventLog.h
@@ -0,0 +1,193 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "core/Core.h"
+#include "FlowFileRecord.h"
+#include "concurrentqueue.h"
+#include "core/Processor.h"
+#include "core/ProcessSession.h"
+#include <pugixml.hpp>
+#include <winevt.h>
+#include <sstream>
+#include <regex>
+#include <codecvt>
+#include "utils/OsUtils.h"
+
+
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace wel {
+
+
+ enum METADATA {
+ LOG_NAME,
+ SOURCE,
+ TIME_CREATED,
+ EVENTID,
+ OPCODE,
+ EVENT_RECORDID,
+ EVENT_TYPE,
+ TASK_CATEGORY,
+ LEVEL,
+ KEYWORDS,
+ USER,
+ COMPUTER,
+ UNKNOWN
+ };
+
+
+// this is a continuous enum so we can rely on the array
+
+typedef std::vector<std::pair<METADATA, std::string>> METADATA_NAMES;
+
+class WindowsEventLogHandler
+{
+public:
+ WindowsEventLogHandler() : metadata_provider_(nullptr){
+ }
+
+ explicit WindowsEventLogHandler(EVT_HANDLE metadataProvider) :
metadata_provider_(metadataProvider) {
+ }
+
+ std::string getEventMessage(EVT_HANDLE eventHandle) const;
+
+
+ EVT_HANDLE getMetadata() const;
+
+private:
+ EVT_HANDLE metadata_provider_;
+};
+
+class WindowsEventLogMetadata {
+public:
+ WindowsEventLogMetadata(EVT_HANDLE metadataProvider, EVT_HANDLE
event_ptr, const std::string &log_name) : metadata_ptr_(metadataProvider),
event_timestamp_(0), event_ptr_(event_ptr), log_name_(log_name) {
+ renderMetadata();
+ }
+
+ virtual std::map<std::string, std::string> getFieldValues() const = 0;
+
+ virtual std::map<std::string, std::string> getIdentifiers() const = 0;
+
+ virtual std::string getMetadata(METADATA metadata) const = 0;
+
+
+ void renderMetadata();
+
+
+ static std::string getMetadataString(METADATA val) {
+ static std::map< METADATA, std::string> map = {
+ {LOG_NAME, "LOG_NAME" },
+ {SOURCE,"SOURCE"},
+ {TIME_CREATED,"TIME_CREATED" },
+ {EVENTID,"EVENTID"},
+ {OPCODE,"OPCODE"},
+ {EVENT_RECORDID,"EVENT_RECORDID"},
+ {EVENT_TYPE,"EVENT_TYPE"},
+ {TASK_CATEGORY, "TASK_CATEGORY"},
+ {LEVEL,"LEVEL"},
+ {KEYWORDS,"KEYWORDS"},
+ {USER,"USER"},
+ {COMPUTER,"COMPUTER"}
+ };
+
+ return map[val];
+ }
+
+
+ static METADATA getMetadataFromString(const std::string &val) {
+ static std::map< std::string, METADATA> map = {
+ {"LOG_NAME",LOG_NAME},
+ {"SOURCE",SOURCE},
+ {"TIME_CREATED",TIME_CREATED },
+ {"EVENTID",EVENTID},
+ {"OPCODE",OPCODE},
+ {"EVENT_RECORDID",EVENT_RECORDID},
+ {"TASK_CATEGORY", TASK_CATEGORY},
+ {"EVENT_TYPE",EVENT_TYPE},
+ {"LEVEL",LEVEL},
+ {"KEYWORDS",KEYWORDS},
+ {"USER",USER},
+ {"COMPUTER",COMPUTER}
+ };
+
+ auto enumVal = map.find(val);
+ if (enumVal != std::end(map)) {
+ return enumVal->second;
+ }
+ else {
+ return METADATA::UNKNOWN;
+ }
+ }
+
+ static std::string getComputerName() {
+ static std::string computer_name;
+ if (computer_name.empty()) {
+ char buff[10248];
+ DWORD size = sizeof(buff);
+ if (GetComputerNameExA(ComputerNameDnsFullyQualified,
buff, &size)) {
+ computer_name = buff;
+ }
+ else {
+ computer_name = "N/A";
+ }
+ }
+ return computer_name;
+ }
+
+protected:
+ std::string log_name_;
+ uint64_t event_timestamp_;
+ std::string event_type_;
+ short event_type_index_;
+ std::string event_timestamp_str_;
+ EVT_HANDLE event_ptr_;
+ EVT_HANDLE metadata_ptr_;
+};
+
+
+class WindowsEventLogHeader {
+public:
+ explicit WindowsEventLogHeader(METADATA_NAMES header_names) :
header_names_(header_names){
+
+ }
+
+ void setDelimiter(const std::string &delim);
+
+ std::string getEventHeader(const WindowsEventLogMetadata * const
metadata) const;
+
+
+
+private:
+
+ inline std::string createDefaultDelimiter(size_t max, size_t length)
const;
+
+ std::string delimiter_;
+ METADATA_NAMES header_names_;
+};
+
+} /* namespace wel */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
diff --git a/libminifi/include/utils/Deleters.h
b/libminifi/include/utils/Deleters.h
new file mode 100644
index 0000000..5a56dfb
--- /dev/null
+++ b/libminifi/include/utils/Deleters.h
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBMINIFI_INCLUDE_UTILS_DELETERS_H
+#define LIBMINIFI_INCLUDE_UTILS_DELETERS_H
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace utils {
+
+class FreeDeleter
+{
+public:
+ void operator()(void* ptr) {
+ // free(null) is guaranteed to be safe, so no need to be
defensive.
+ free(ptr);
+ }
+};
+
+} /* namespace utils */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+#endif // LIBMINIFI_INCLUDE_UTILS_DELETERS_H
\ No newline at end of file
