This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit f65f14543941935d84a5363b278fcb30949f01f0 Author: Ferenc Gerlits <[email protected]> AuthorDate: Mon Sep 8 15:32:38 2025 +0200 MINIFICPP-2589 Refactor ConsumeWindowsEventLog * lots of renaming stuff * use magic_enum instead of hand-rolled enum parsing * some inlining / pulling out of methods * unify the printing of Windows error messages using fmt::format etc Closes #2007 Signed-off-by: Marton Szasz <[email protected]> --- .../windows-event-log/ConsumeWindowsEventLog.cpp | 275 ++++++++++----------- .../windows-event-log/ConsumeWindowsEventLog.h | 59 ++--- .../windows-event-log/tests/BookmarkTests.cpp | 11 +- extensions/windows-event-log/tests/CMakeLists.txt | 2 +- .../tests/ConsumeWindowsEventLogTests.cpp | 2 +- .../tests/MetadataWalkerTests.cpp | 130 +++++----- .../tests/StringSplitterTests.cpp | 45 ++++ .../windows-event-log/tests/WindowsErrorTests.cpp | 34 +++ .../windows-event-log/{ => wel}/Bookmark.cpp | 29 +-- extensions/windows-event-log/{ => wel}/Bookmark.h | 15 +- extensions/windows-event-log/wel/EventPath.h | 2 +- .../windows-event-log/wel/MetadataWalker.cpp | 60 +++-- extensions/windows-event-log/wel/MetadataWalker.h | 21 +- extensions/windows-event-log/wel/StringSplitter.h | 56 +++++ extensions/windows-event-log/wel/UniqueEvtHandle.h | 15 +- .../wel/{UniqueEvtHandle.h => WindowsError.h} | 31 +-- .../windows-event-log/wel/WindowsEventLog.cpp | 28 +-- extensions/windows-event-log/wel/WindowsEventLog.h | 77 ++---- libminifi/test/libtest/unit/Catch.h | 1 + libminifi/test/libtest/unit/TestUtils.h | 10 +- 20 files changed, 472 insertions(+), 431 deletions(-) diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index e5bce24e3..cb62bc03c 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -1,7 +1,4 @@ /** - * @file ConsumeWindowsEventLog.cpp - * ConsumeWindowsEventLog class declaration - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,31 +16,30 @@ */ #include "ConsumeWindowsEventLog.h" + #include <cstdio> #include <vector> #include <tuple> #include <utility> #include <map> -#include <sstream> #include <string> #include <memory> +#include "wel/Bookmark.h" #include "wel/LookupCacher.h" #include "wel/MetadataWalker.h" +#include "wel/StringSplitter.h" #include "wel/XMLString.h" #include "wel/JSONUtils.h" -#include "wel/UniqueEvtHandle.h" +#include "wel/WindowsError.h" -#include "io/BufferStream.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" -#include "Bookmark.h" #include "utils/Deleters.h" #include "core/logging/LoggerFactory.h" #include "utils/gsl.h" -#include "utils/OsUtils.h" #include "utils/RegexUtils.h" #include "utils/StringUtils.h" #include "utils/UnicodeConversion.h" @@ -54,9 +50,31 @@ using namespace std::literals::chrono_literals; -namespace org::apache::nifi::minifi::processors { +namespace { +auto createTimer() { + return [start_time = std::chrono::steady_clock::now()] { + return std::chrono::steady_clock::now() - start_time; + }; +} +} // namespace + +namespace org::apache::nifi::minifi::wel { +std::function<bool(std::string_view)> parseSidMatcher(const std::optional<std::string>& sid_matcher) { + if (!sid_matcher || sid_matcher->empty()) { + return [](std::string_view){ return false; }; + } + + if (std::smatch match; utils::regexMatch(*sid_matcher, match, utils::Regex{R"_(\.\*(\w+))_"})) { + std::string suffix = match[1]; + return [suffix](std::string_view field_name) { return utils::string::endsWith(field_name, suffix); }; + } -const int EVT_NEXT_TIMEOUT_MS = 500; + utils::Regex sid_matcher_regex{*sid_matcher}; + return [sid_matcher_regex](std::string_view field_name) { return utils::regexMatch(field_name, sid_matcher_regex); }; +} +} // namespace org::apache::nifi::minifi::wel + +namespace org::apache::nifi::minifi::processors { ConsumeWindowsEventLog::ConsumeWindowsEventLog(core::ProcessorMetadata metadata) : core::ProcessorImpl{std::move(metadata)} { @@ -65,8 +83,7 @@ ConsumeWindowsEventLog::ConsumeWindowsEventLog(core::ProcessorMetadata metadata) if (GetComputerName(buff, &size)) { computerName_ = buff; } else { - auto last_error = utils::OsUtils::windowsErrorToErrorCode(GetLastError()); - logger_->log_error("{}: {}", last_error, last_error.message()); + logger_->log_error("GetComputerName failed due to {}", wel::getLastError()); } } @@ -78,7 +95,7 @@ void ConsumeWindowsEventLog::notifyStop() { if (FreeLibrary(hMsobjsDll_)) { hMsobjsDll_ = nullptr; } else { - LOG_LAST_ERROR(LoadLibrary); + logger_->log_error("FreeLibrary failed due to {}", wel::getLastError()); } } logger_->log_trace("finish notifyStop"); @@ -95,14 +112,18 @@ void ConsumeWindowsEventLog::initialize() { setSupportedRelationships(Relationships); } -bool ConsumeWindowsEventLog::insertHeaderName(wel::METADATA_NAMES &header, const std::string &key, const std::string & value) { - wel::METADATA name = wel::WindowsEventLogMetadata::getMetadataFromString(key); +wel::HeaderNames ConsumeWindowsEventLog::createHeaderNames(const std::optional<std::string>& event_header_property) const { + if (!event_header_property) { return {}; } - if (name != wel::METADATA::UNKNOWN) { - header.emplace_back(std::make_pair(name, value)); - return true; - } - return false; + wel::HeaderNames header_names; + wel::splitCommaSeparatedKeyValuePairs(*event_header_property, [this, &header_names](std::string_view key, std::string_view value) { + if (auto metadata = magic_enum::enum_cast<wel::Metadata>(key)) { + header_names.emplace_back(*metadata, value); + } else { + logger_->log_error("{} is an invalid key for the header map", key); + } + }); + return header_names; } void ConsumeWindowsEventLog::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { @@ -113,47 +134,26 @@ void ConsumeWindowsEventLog::onSchedule(core::ProcessContext& context, core::Pro resolve_as_attributes_ = utils::parseBoolProperty(context, ResolveAsAttributes); apply_identifier_function_ = utils::parseBoolProperty(context, IdentifierFunction); - header_delimiter_ = utils::parseOptionalProperty(context, EventHeaderDelimiter); batch_commit_size_ = utils::parseU64Property(context, BatchCommitSize); + header_names_ = createHeaderNames(utils::parseOptionalProperty(context, EventHeader)); + sid_matcher_ = wel::parseSidMatcher(utils::parseOptionalProperty(context, IdentifierMatcher)); + output_format_ = utils::parseEnumProperty<wel::OutputFormat>(context, OutputFormatProperty); + json_format_ = utils::parseEnumProperty<wel::JsonFormat>(context, JsonFormatProperty); - header_names_.clear(); - if (auto header = context.getProperty(EventHeader)) { - auto keyValueSplit = utils::string::split(*header, ","); - for (const auto &kv : keyValueSplit) { - auto splitKeyAndValue = utils::string::split(kv, "="); - if (splitKeyAndValue.size() == 2) { - auto key = utils::string::trim(splitKeyAndValue.at(0)); - auto value = utils::string::trim(splitKeyAndValue.at(1)); - if (!insertHeaderName(header_names_, key, value)) { - logger_->log_error("{} is an invalid key for the header map", key); - } - } else if (splitKeyAndValue.size() == 1) { - auto key = utils::string::trim(splitKeyAndValue.at(0)); - if (!insertHeaderName(header_names_, key, "")) { - logger_->log_error("{} is an invalid key for the header map", key); - } - } - } - } - - sid_matcher_ = cwel::parseSidMatcher(utils::parseOptionalProperty(context, IdentifierMatcher)); - output_format_ = utils::parseEnumProperty<cwel::OutputFormat>(context, OutputFormatProperty); - json_format_ = utils::parseEnumProperty<cwel::JsonFormat>(context, JsonFormatProperty); - - if (output_format_ != cwel::OutputFormat::Plaintext && !hMsobjsDll_) { + if (output_format_ != wel::OutputFormat::Plaintext && !hMsobjsDll_) { char systemDir[MAX_PATH]; if (GetSystemDirectory(systemDir, sizeof(systemDir))) { hMsobjsDll_ = LoadLibrary((systemDir + std::string("\\msobjs.dll")).c_str()); if (!hMsobjsDll_) { - LOG_LAST_ERROR(LoadLibrary); + logger_->log_error("LoadLibrary failed due to {}", wel::getLastError()); } } else { - LOG_LAST_ERROR(GetSystemDirectory); + logger_->log_error("GetSystemDirectory failed due to {}", wel::getLastError()); } } - path_ = wel::EventPath{context.getProperty(Channel).value()}; + path_ = wel::EventPath{utils::parseProperty(context, Channel)}; if (path_.kind() == wel::EventPath::Kind::FILE) { logger_->log_debug("Using saved log file as log source"); } else { @@ -161,21 +161,10 @@ void ConsumeWindowsEventLog::onSchedule(core::ProcessContext& context, core::Pro } std::string query = context.getProperty(Query).value_or(""); - wstr_query_ = std::wstring(query.begin(), query.end()); - - bool processOldEvents = utils::parseBoolProperty(context, ProcessOldEvents); + wstr_query_ = utils::to_wstring(query); if (!bookmark_) { - std::string bookmarkDir = context.getProperty(BookmarkRootDirectory).value_or(""); - if (bookmarkDir.empty()) { - logger_->log_error("State Directory is empty"); - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "State Directory is empty"); - } - bookmark_ = std::make_unique<Bookmark>(path_, wstr_query_, bookmarkDir, getUUID(), processOldEvents, state_manager_, logger_); - if (!*bookmark_) { - bookmark_.reset(); - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Bookmark is empty"); - } + bookmark_ = createBookmark(context); } max_buffer_size_ = utils::parseDataSizeProperty(context, MaxBufferSize); @@ -188,26 +177,26 @@ void ConsumeWindowsEventLog::onSchedule(core::ProcessContext& context, core::Pro logger_->log_trace("Successfully configured CWEL"); } -std::function<bool(std::string_view)> cwel::parseSidMatcher(const std::optional<std::string>& sid_matcher) { - if (!sid_matcher || sid_matcher->empty()) { - return [](std::string_view){ return false; }; +std::unique_ptr<wel::Bookmark> ConsumeWindowsEventLog::createBookmark(const core::ProcessContext& context) const { + std::string bookmark_dir = context.getProperty(BookmarkRootDirectory).value_or(""); + if (bookmark_dir.empty()) { + logger_->log_error("State Directory is empty"); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "State Directory is empty"); } - - if (std::smatch match; utils::regexMatch(*sid_matcher, match, utils::Regex{R"_(\.\*(\w+))_"})) { - std::string suffix = match[1]; - return [suffix](std::string_view field_name) { return utils::string::endsWith(field_name, suffix); }; + bool process_old_events = utils::parseBoolProperty(context, ProcessOldEvents); + auto bookmark = std::make_unique<wel::Bookmark>(path_, wstr_query_, bookmark_dir, getUUID(), process_old_events, state_manager_, logger_); + if (!bookmark->isValid()) { + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Bookmark is empty"); } - - utils::Regex sid_matcher_regex{*sid_matcher}; - return [sid_matcher_regex](std::string_view field_name) { return utils::regexMatch(field_name, sid_matcher_regex); }; + return bookmark; } bool ConsumeWindowsEventLog::commitAndSaveBookmark(const std::wstring &bookmark_xml, core::ProcessContext& context, core::ProcessSession& session) { { - const TimeDiff time_diff; + const auto time_diff = createTimer(); session.commit(); context.getStateManager()->beginTransaction(); - logger_->log_debug("processQueue commit took {}", time_diff()); + logger_->log_debug("ConsumeWindowsEventLog: commit took {}", time_diff()); } if (!bookmark_->saveBookmarkXml(bookmark_xml)) { @@ -222,17 +211,17 @@ bool ConsumeWindowsEventLog::commitAndSaveBookmark(const std::wstring &bookmark_ return true; } -std::tuple<size_t, std::wstring> ConsumeWindowsEventLog::processEventLogs(core::ProcessSession& session, const EVT_HANDLE& event_query_results) { +std::tuple<size_t, std::wstring> ConsumeWindowsEventLog::processEventLogs(core::ProcessSession& session, EVT_HANDLE event_query_results) { + static constexpr DWORD timeout_milliseconds = 500; size_t processed_event_count = 0; std::wstring bookmark_xml; logger_->log_trace("Enumerating the events in the result set after the bookmarked event."); while (processed_event_count < batch_commit_size_ || batch_commit_size_ == 0) { - EVT_HANDLE next_event{}; + EVT_HANDLE next_event_handle{}; DWORD handles_set_count{}; - if (!EvtNext(event_query_results, 1, &next_event, EVT_NEXT_TIMEOUT_MS, 0, &handles_set_count)) { + if (!EvtNext(event_query_results, 1, &next_event_handle, timeout_milliseconds, 0, &handles_set_count)) { if (ERROR_NO_MORE_ITEMS != GetLastError()) { - auto last_error = utils::OsUtils::windowsErrorToErrorCode(GetLastError()); - logger_->log_error("Failed to get next event: {}: {}", last_error, last_error.message()); + logger_->log_error("Failed to get next event due to {}", wel::getLastError()); continue; /* According to MS this iteration should only end when the return value is false AND the error code is NO_MORE_ITEMS. See the following page for further details: @@ -240,22 +229,21 @@ std::tuple<size_t, std::wstring> ConsumeWindowsEventLog::processEventLogs(core:: } break; } - - const auto guard_next_event = gsl::finally([next_event]() { EvtClose(next_event); }); - logger_->log_trace("Succesfully got the next event, performing event rendering"); - auto event_render = createEventRender(next_event); - if (!event_render) { - logger_->log_error("{}", event_render.error()); + wel::unique_evt_handle next_event{next_event_handle}; + logger_->log_trace("Successfully got the next event, performing event rendering"); + auto processed_event = processEvent(next_event.get()); + if (!processed_event) { + logger_->log_error("error processing event: {}", processed_event.error()); continue; } - auto new_bookmark_xml = bookmark_->getNewBookmarkXml(next_event); + auto new_bookmark_xml = bookmark_->getNewBookmarkXml(next_event.get()); if (!new_bookmark_xml) { - logger_->log_error("{}", new_bookmark_xml.error()); + logger_->log_error("error getting the new bookmark: {}", new_bookmark_xml.error()); continue; } bookmark_xml = std::move(*new_bookmark_xml); processed_event_count++; - putEventRenderFlowFileToSession(*event_render, session); + createAndCommitFlowFile(*processed_event, session); } logger_->log_trace("Finished enumerating events."); return std::make_tuple(processed_event_count, bookmark_xml); @@ -277,14 +265,14 @@ void ConsumeWindowsEventLog::onTrigger(core::ProcessContext& context, core::Proc logger_->log_trace("CWEL onTrigger"); size_t processed_event_count = 0; - const TimeDiff time_diff; + const auto time_diff = createTimer(); const auto timeGuard = gsl::finally([&]() { logger_->log_debug("processed {} Events in {}", processed_event_count, time_diff()); }); wel::unique_evt_handle event_query_results{EvtQuery(nullptr, path_.wstr().c_str(), wstr_query_.c_str(), path_.getQueryFlags())}; if (!event_query_results) { - LOG_LAST_ERROR(EvtQuery); + logger_->log_error("EvtQuery failed due to {}", wel::getLastError()); context.yield(); return; } @@ -300,7 +288,7 @@ void ConsumeWindowsEventLog::onTrigger(core::ProcessContext& context, core::Proc } if (!EvtSeek(event_query_results.get(), 1, bookmark_handle, 0, EvtSeekRelativeToBookmark)) { - LOG_LAST_ERROR(EvtSeek); + logger_->log_error("EvtSeek failed due to {}", wel::getLastError()); context.yield(); return; } @@ -316,22 +304,23 @@ void ConsumeWindowsEventLog::onTrigger(core::ProcessContext& context, core::Proc } } -wel::WindowsEventLogHandler& ConsumeWindowsEventLog::getEventLogHandler(const std::string & name) { - logger_->log_trace("Getting Event Log Handler corresponding to {}", name.c_str()); - auto provider = providers_.find(name); - if (provider != std::end(providers_)) { - logger_->log_trace("Found the handler"); - return provider->second; +wel::WindowsEventLogProvider& ConsumeWindowsEventLog::getEventLogProvider(const std::string& name) { + logger_->log_trace("Getting Event Log Provider corresponding to {}", name); + auto it = providers_.find(name); + if (it != std::end(providers_)) { + logger_->log_trace("Found cached event log provider"); + return it->second; } - std::wstring temp_wstring = std::wstring(name.begin(), name.end()); + std::wstring temp_wstring = utils::to_wstring(name); LPCWSTR widechar = temp_wstring.c_str(); auto opened_publisher_metadata_provider = EvtOpenPublisherMetadata(nullptr, widechar, nullptr, 0, 0); - if (!opened_publisher_metadata_provider) - logger_->log_warn("EvtOpenPublisherMetadata failed due to {}", utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message()); + if (!opened_publisher_metadata_provider) { + logger_->log_warn("EvtOpenPublisherMetadata failed due to {}", wel::getLastError()); + } providers_.emplace(name, opened_publisher_metadata_provider); - logger_->log_info("Handler not found for {}, creating. Number of cached handlers: {}", name, providers_.size()); + logger_->log_info("Created new Windows event log provider for '{}'. Number of cached providers: {}", name, providers_.size()); return providers_[name]; } @@ -392,7 +381,7 @@ void ConsumeWindowsEventLog::substituteXMLPercentageItems(pugi::xml_document& do // Add "" to xmlPercentageItemsResolutions_ - don't need to call FormaMessage for this 'key' again. xmlPercentageItemsResolutions_.insert({key, ""}); - logger_->log_error("!FormatMessage error: {:#x}. '{}' is not found in msobjs.dll.", GetLastError(), key.c_str()); + logger_->log_error("FormatMessage failed due to {}. '{}' is not found in msobjs.dll.", wel::getLastError(), key); } } else { value = it->second; @@ -431,30 +420,27 @@ nonstd::expected<std::string, std::string> ConsumeWindowsEventLog::renderEventAs DWORD used = 0; DWORD propertyCount = 0; if (!EvtRender(nullptr, event_handle, EvtRenderEventXml, size, buf.get(), &used, &propertyCount)) { - DWORD last_error = GetLastError(); - if (ERROR_INSUFFICIENT_BUFFER != last_error) { - std::string error_message = fmt::format("EvtRender failed due to {}", utils::OsUtils::windowsErrorToErrorCode(last_error).message()); - return nonstd::make_unexpected(error_message); + if (ERROR_INSUFFICIENT_BUFFER != GetLastError()) { + return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", wel::getLastError())); } if (used > max_buffer_size_) { - std::string error_message = fmt::format("Dropping event because it couldn't be rendered within {} bytes.", max_buffer_size_); - return nonstd::make_unexpected(error_message); + return nonstd::make_unexpected(fmt::format("Dropping event because it couldn't be rendered within {} bytes.", max_buffer_size_)); } size = used; buf.reset((LPWSTR) malloc(size)); - if (!buf) + if (!buf) { return nonstd::make_unexpected("malloc failed"); + } if (!EvtRender(nullptr, event_handle, EvtRenderEventXml, size, buf.get(), &used, &propertyCount)) { - std::string error_message = fmt::format("EvtRender failed due to {}", utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message()); - return nonstd::make_unexpected(error_message); + return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", wel::getLastError())); } } logger_->log_trace("Event rendered with size {}", used); return utils::to_string(std::wstring{buf.get()}); } -nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent) { - auto event_as_xml = renderEventAsXml(hEvent); +nonstd::expected<wel::ProcessedEvent, std::string> ConsumeWindowsEventLog::processEvent(EVT_HANDLE event_handle) { + auto event_as_xml = renderEventAsXml(event_handle); if (!event_as_xml) return nonstd::make_unexpected(event_as_xml.error()); @@ -462,11 +448,11 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE if (!doc.load_string(event_as_xml->c_str())) return nonstd::make_unexpected("Invalid XML produced"); - cwel::EventRender result; + wel::ProcessedEvent result; // this is a well known path. std::string provider_name = doc.child("Event").child("System").child("Provider").attribute("Name").value(); - wel::WindowsEventLogMetadataImpl metadata{getEventLogHandler(provider_name), hEvent}; + wel::WindowsEventLogMetadataImpl metadata{getEventLogProvider(provider_name), event_handle}; wel::MetadataWalker walker{metadata, path_.str(), !resolve_as_attributes_, apply_identifier_function_, sid_matcher_, userIdToUsernameFunction()}; // resolve the event metadata @@ -474,11 +460,11 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE logger_->log_trace("Finish doc traversing, performing writing..."); - if (output_format_ == cwel::OutputFormat::Plaintext || output_format_ == cwel::OutputFormat::Both) { + if (output_format_ == wel::OutputFormat::Plaintext || output_format_ == wel::OutputFormat::Both) { logger_->log_trace("Writing event in plain text"); - auto& handler = getEventLogHandler(provider_name); - auto event_message = handler.getEventMessage(hEvent); + auto& provider = getEventLogProvider(provider_name); + auto event_message = provider.getEventMessage(event_handle); if (event_message) { for (const auto& map_entry : walker.getIdentifiers()) { @@ -492,7 +478,7 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE std::string_view payload_name = event_message ? "Message" : "Error"; wel::WindowsEventLogHeader log_header(header_names_, header_delimiter_, payload_name.size()); - result.plaintext = log_header.getEventHeader([&walker](wel::METADATA metadata) { return walker.getMetadata(metadata); }); + result.plaintext = log_header.getEventHeader([&walker](wel::Metadata metadata) { return walker.getMetadata(metadata); }); result.plaintext += payload_name; result.plaintext += log_header.getDelimiterFor(payload_name.size()); result.plaintext += event_message.has_value() ? *event_message : event_message.error().message(); @@ -500,7 +486,7 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE logger_->log_trace("Finish writing in plain text"); } - if (output_format_ != cwel::OutputFormat::Plaintext) { + if (output_format_ != wel::OutputFormat::Plaintext) { substituteXMLPercentageItems(doc); logger_->log_trace("Finish substituting %% in XML"); } @@ -509,7 +495,7 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE result.matched_fields = walker.getFieldValues(); } - if (output_format_ == cwel::OutputFormat::XML || output_format_ == cwel::OutputFormat::Both) { + if (output_format_ == wel::OutputFormat::XML || output_format_ == wel::OutputFormat::Both) { logger_->log_trace("Writing event in XML"); wel::XmlString writer; @@ -520,21 +506,21 @@ nonstd::expected<cwel::EventRender, std::string> ConsumeWindowsEventLog::createE logger_->log_trace("Finish writing in XML"); } - if (output_format_ == cwel::OutputFormat::JSON) { + if (output_format_ == wel::OutputFormat::JSON) { switch (json_format_) { - case cwel::JsonFormat::Raw: { + case wel::JsonFormat::Raw: { logger_->log_trace("Writing event in raw JSON"); result.json = wel::jsonToString(wel::toRawJSON(doc)); logger_->log_trace("Finish writing in raw JSON"); break; } - case cwel::JsonFormat::Simple: { + case wel::JsonFormat::Simple: { logger_->log_trace("Writing event in simple JSON"); result.json = wel::jsonToString(wel::toSimpleJSON(doc)); logger_->log_trace("Finish writing in simple JSON"); break; } - case cwel::JsonFormat::Flattened: { + case wel::JsonFormat::Flattened: { logger_->log_trace("Writing event in flattened JSON"); result.json = wel::jsonToString(wel::toFlattenedJSON(doc)); logger_->log_trace("Finish writing in flattened JSON"); @@ -573,21 +559,22 @@ void ConsumeWindowsEventLog::refreshTimeZoneData() { break; } - tzbias *= -1; // WinApi specifies UTC = localtime + bias, but we need offset from UTC - std::stringstream tzoffset; - tzoffset << (tzbias >= 0 ? '+' : '-') << std::setfill('0') << std::setw(2) << std::abs(tzbias) / 60 - << ":" << std::setfill('0') << std::setw(2) << std::abs(tzbias) % 60; - timezone_name_ = utils::to_string(tzstr); - timezone_offset_ = tzoffset.str(); + + tzbias *= -1; // WinApi specifies UTC = localtime + bias, but we need offset from UTC + timezone_offset_ = fmt::format("{}{:02}:{:02}", tzbias >= 0 ? '+' : '-', std::abs(tzbias) / 60, std::abs(tzbias) % 60); logger_->log_trace("Timezone name: {}, offset: {}", timezone_name_, timezone_offset_); } -void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const cwel::EventRender& eventRender, core::ProcessSession& session) const { - auto commitFlowFile = [&] (const std::string& content, const std::string& mimeType) { +void ConsumeWindowsEventLog::createAndCommitFlowFile(const wel::ProcessedEvent& processed_event, core::ProcessSession& session) const { + auto commit_flow_file = [&] (const std::string& content, const std::string& mimeType) { auto flow_file = session.create(); - addMatchedFieldsAsAttributes(eventRender, session, flow_file); + for (const auto& [key, value] : processed_event.matched_fields) { + if (!value.empty()) { + session.putAttribute(*flow_file, key, value); + } + } session.writeBuffer(flow_file, content); session.putAttribute(*flow_file, core::SpecialFlowAttribute::MIME_TYPE, mimeType); session.putAttribute(*flow_file, "timezone.name", timezone_name_); @@ -596,27 +583,19 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const cwel::EventRe session.transfer(flow_file, Success); }; - if (output_format_ == cwel::OutputFormat::XML || output_format_ == cwel::OutputFormat::Both) { + if (output_format_ == wel::OutputFormat::XML || output_format_ == wel::OutputFormat::Both) { logger_->log_trace("Writing rendered XML to a flow file"); - commitFlowFile(eventRender.xml, "application/xml"); + commit_flow_file(processed_event.xml, "application/xml"); } - if (output_format_ == cwel::OutputFormat::Plaintext || output_format_ == cwel::OutputFormat::Both) { + if (output_format_ == wel::OutputFormat::Plaintext || output_format_ == wel::OutputFormat::Both) { logger_->log_trace("Writing rendered plain text to a flow file"); - commitFlowFile(eventRender.plaintext, "text/plain"); + commit_flow_file(processed_event.plaintext, "text/plain"); } - if (output_format_ == cwel::OutputFormat::JSON) { + if (output_format_ == wel::OutputFormat::JSON) { logger_->log_trace("Writing rendered {} JSON to a flow file", magic_enum::enum_name(json_format_)); - commitFlowFile(eventRender.json, "application/json"); - } -} - -void ConsumeWindowsEventLog::addMatchedFieldsAsAttributes(const cwel::EventRender& eventRender, core::ProcessSession& session, const std::shared_ptr<core::FlowFile>& flowFile) const { - for (const auto &fieldMapping : eventRender.matched_fields) { - if (!fieldMapping.second.empty()) { - session.putAttribute(*flowFile, fieldMapping.first, fieldMapping.second); - } + commit_flow_file(processed_event.json, "application/json"); } } diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index 3c48a3220..b2760c1d0 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -1,7 +1,4 @@ /** - * @file ConsumeWindowsEventLog.h - * ConsumeWindowsEventLog class declaration - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -52,10 +49,8 @@ #include "utils/Export.h" #include "utils/RegexUtils.h" -namespace org::apache::nifi::minifi::processors { - -namespace cwel { -struct EventRender { +namespace org::apache::nifi::minifi::wel { +struct ProcessedEvent { std::map<std::string, std::string> matched_fields; std::string xml; std::string plaintext; @@ -76,9 +71,11 @@ enum class JsonFormat { }; std::function<bool(std::string_view)> parseSidMatcher(const std::optional<std::string>& sid_matcher); -} // namespace cwel class Bookmark; +} // namespace org::apache::nifi::minifi::wel + +namespace org::apache::nifi::minifi::processors { class ConsumeWindowsEventLog : public core::ProcessorImpl { public: @@ -145,16 +142,16 @@ class ConsumeWindowsEventLog : public core::ProcessorImpl { .withDescription("Comma seperated list of key/value pairs with the following keys LOG_NAME, SOURCE, TIME_CREATED,EVENT_RECORDID," "EVENTID,TASK_CATEGORY,LEVEL,KEYWORDS,USER,COMPUTER, and EVENT_TYPE. Eliminating fields will remove them from the header.") .build(); - EXTENSIONAPI static constexpr auto OutputFormatProperty = core::PropertyDefinitionBuilder<magic_enum::enum_count<cwel::OutputFormat>()>::createProperty("Output Format") + EXTENSIONAPI static constexpr auto OutputFormatProperty = core::PropertyDefinitionBuilder<magic_enum::enum_count<wel::OutputFormat>()>::createProperty("Output Format") .isRequired(true) - .withDefaultValue(magic_enum::enum_name(cwel::OutputFormat::XML)) - .withAllowedValues(magic_enum::enum_names<cwel::OutputFormat>()) + .withDefaultValue(magic_enum::enum_name(wel::OutputFormat::XML)) + .withAllowedValues(magic_enum::enum_names<wel::OutputFormat>()) .withDescription("The format of the output flow files.") .build(); - EXTENSIONAPI static constexpr auto JsonFormatProperty = core::PropertyDefinitionBuilder<magic_enum::enum_count<cwel::JsonFormat>()>::createProperty("JSON Format") + EXTENSIONAPI static constexpr auto JsonFormatProperty = core::PropertyDefinitionBuilder<magic_enum::enum_count<wel::JsonFormat>()>::createProperty("JSON Format") .isRequired(true) - .withDefaultValue(magic_enum::enum_name(cwel::JsonFormat::Simple)) - .withAllowedValues(magic_enum::enum_names<cwel::JsonFormat>()) + .withDefaultValue(magic_enum::enum_name(wel::JsonFormat::Simple)) + .withAllowedValues(magic_enum::enum_names<wel::JsonFormat>()) .withDescription("Set the json format type. Only applicable if Output Format is set to 'JSON'") .build(); EXTENSIONAPI static constexpr auto BatchCommitSize = core::PropertyDefinitionBuilder<>::createProperty("Batch Commit Size") @@ -216,31 +213,19 @@ class ConsumeWindowsEventLog : public core::ProcessorImpl { private: void refreshTimeZoneData(); - void putEventRenderFlowFileToSession(const cwel::EventRender& eventRender, core::ProcessSession& session) const; - wel::WindowsEventLogHandler& getEventLogHandler(const std::string& name); - static bool insertHeaderName(wel::METADATA_NAMES& header, const std::string& key, const std::string& value); - nonstd::expected<cwel::EventRender, std::string> createEventRender(EVT_HANDLE eventHandle); + void createAndCommitFlowFile(const wel::ProcessedEvent& processed_event, core::ProcessSession& session) const; + wel::WindowsEventLogProvider& getEventLogProvider(const std::string& name); + wel::HeaderNames createHeaderNames(const std::optional<std::string>& event_header_property) const; + nonstd::expected<wel::ProcessedEvent, std::string> processEvent(EVT_HANDLE event_handle); void substituteXMLPercentageItems(pugi::xml_document& doc); std::function<std::string(const std::string&)> userIdToUsernameFunction() const; - nonstd::expected<std::string, std::string> renderEventAsXml(EVT_HANDLE event_handle); - - struct TimeDiff { - auto operator()() const { - return std::chrono::steady_clock::now() - time_; - } - const decltype(std::chrono::steady_clock::now()) time_ = std::chrono::steady_clock::now(); - }; - bool commitAndSaveBookmark(const std::wstring& bookmarkXml, core::ProcessContext& context, core::ProcessSession& session); - - std::tuple<size_t, std::wstring> processEventLogs(core::ProcessSession& session, - const EVT_HANDLE& event_query_results); - - void addMatchedFieldsAsAttributes(const cwel::EventRender &eventRender, core::ProcessSession &session, const std::shared_ptr<core::FlowFile> &flowFile) const; + std::tuple<size_t, std::wstring> processEventLogs(core::ProcessSession& session, EVT_HANDLE event_query_results); + std::unique_ptr<wel::Bookmark> createBookmark(const core::ProcessContext& context) const; core::StateManager* state_manager_{nullptr}; - wel::METADATA_NAMES header_names_; + wel::HeaderNames header_names_; std::optional<std::string> header_delimiter_; wel::EventPath path_; std::wstring wstr_query_; @@ -250,14 +235,14 @@ class ConsumeWindowsEventLog : public core::ProcessorImpl { std::string provenanceUri_; std::string computerName_; uint64_t max_buffer_size_{}; - std::map<std::string, wel::WindowsEventLogHandler> providers_; + std::map<std::string, wel::WindowsEventLogProvider> providers_; uint64_t batch_commit_size_{}; bool cache_sid_lookups_ = true; - cwel::OutputFormat output_format_; - cwel::JsonFormat json_format_; + wel::OutputFormat output_format_; + wel::JsonFormat json_format_; - std::unique_ptr<Bookmark> bookmark_; + std::unique_ptr<wel::Bookmark> bookmark_; std::mutex on_trigger_mutex_; std::unordered_map<std::string, std::string> xmlPercentageItemsResolutions_; HMODULE hMsobjsDll_{}; diff --git a/extensions/windows-event-log/tests/BookmarkTests.cpp b/extensions/windows-event-log/tests/BookmarkTests.cpp index a6151ee23..cd1f4edf5 100644 --- a/extensions/windows-event-log/tests/BookmarkTests.cpp +++ b/extensions/windows-event-log/tests/BookmarkTests.cpp @@ -15,18 +15,17 @@ * limitations under the License. */ -#include "Bookmark.h" - #include <memory> #include <regex> #include "unit/TestBase.h" #include "unit/Catch.h" #include "utils/gsl.h" +#include "wel/Bookmark.h" #include "wel/UniqueEvtHandle.h" #include "CWELTestUtils.h" -using Bookmark = org::apache::nifi::minifi::processors::Bookmark; +using Bookmark = org::apache::nifi::minifi::wel::Bookmark; using unique_evt_handle = org::apache::nifi::minifi::wel::unique_evt_handle; using IdGenerator = org::apache::nifi::minifi::utils::IdGenerator; @@ -68,7 +67,7 @@ std::wstring bookmarkHandleAsXml(EVT_HANDLE event) { std::wstring bookmarkAsXml(const std::unique_ptr<Bookmark>& bookmark) { REQUIRE(bookmark); - REQUIRE(*bookmark); + REQUIRE(bookmark->isValid()); return bookmarkHandleAsXml(bookmark->getBookmarkHandleFromXML()); } @@ -108,7 +107,7 @@ TEST_CASE("Bookmark constructor works", "[create]") { auto state_manager = test_plan->getStateStorage()->getStateManager(uuid); std::unique_ptr<Bookmark> bookmark = createBookmark(*test_plan, APPLICATION_CHANNEL, uuid, state_manager.get()); REQUIRE(bookmark); - REQUIRE(*bookmark); + REQUIRE(bookmark->isValid()); std::wregex pattern{L"<BookmarkList Direction='backward'>\r\n" L" <Bookmark Channel='Application' RecordId='\\d+' IsCurrent='true'/>\r\n" @@ -130,7 +129,7 @@ TEST_CASE("Bookmark constructor works for log file path", "[create]") { auto state_manager = test_plan->getStateStorage()->getStateManager(uuid); std::unique_ptr<Bookmark> bookmark = createBookmark(*test_plan, L"SavedLog:" + log_file.wstring(), uuid, state_manager.get()); REQUIRE(bookmark); - REQUIRE(*bookmark); + REQUIRE(bookmark->isValid()); std::string log_file_str = log_file.string(); utils::string::replaceAll(log_file_str, "\\", "\\\\"); diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 07c40712f..d0ba91a4e 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -set(WEL_TESTS "BookmarkTests.cpp" "ConsumeWindowsEventLogTests.cpp" "LookupCacherTests.cpp" "MetadataWalkerTests.cpp") +set(WEL_TESTS "BookmarkTests.cpp" "ConsumeWindowsEventLogTests.cpp" "LookupCacherTests.cpp" "StringSplitterTests.cpp" "MetadataWalkerTests.cpp" "WindowsErrorTests.cpp") if (TEST_CUSTOM_WEL_PROVIDER) execute_process(COMMAND "${CMAKE_CURRENT_LIST_DIR}/custom-provider/generate-and-register.bat" diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index 353e91555..a1ea911c9 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -543,7 +543,7 @@ TEST_CASE("ConsumeWindowsEventLog can process events from a log file", "[cwel][l TEST_CASE("ConsumeWindowsEventLog::parseSidMatcher works correctly") { const auto verify = [](const std::optional<std::string>& sid_matcher, std::string_view field_name, bool result) { - const auto matcher_function = minifi::processors::cwel::parseSidMatcher(sid_matcher); + const auto matcher_function = minifi::wel::parseSidMatcher(sid_matcher); CHECK(matcher_function(field_name) == result); }; diff --git a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp index 678e0c768..84af243b5 100644 --- a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp +++ b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp @@ -28,21 +28,21 @@ #include "wel/XMLString.h" #include "pugixml.hpp" -using METADATA = org::apache::nifi::minifi::wel::METADATA; +using Metadata = org::apache::nifi::minifi::wel::Metadata; using MetadataWalker = org::apache::nifi::minifi::wel::MetadataWalker; -using WindowsEventLogHandler = org::apache::nifi::minifi::wel::WindowsEventLogHandler; +using WindowsEventLogProvider = org::apache::nifi::minifi::wel::WindowsEventLogProvider; using WindowsEventLogMetadata = org::apache::nifi::minifi::wel::WindowsEventLogMetadata; using WindowsEventLogMetadataImpl = org::apache::nifi::minifi::wel::WindowsEventLogMetadataImpl; using XmlString = org::apache::nifi::minifi::wel::XmlString; namespace { -auto none_matcher = minifi::processors::cwel::parseSidMatcher(std::nullopt); +auto none_matcher = minifi::wel::parseSidMatcher(std::nullopt); -std::string updateXmlMetadata(const std::string &xml, EVT_HANDLE metadata_ptr, EVT_HANDLE event_ptr, bool update_xml, bool resolve, +std::string updateXmlMetadata(const std::string &xml, EVT_HANDLE provider_handle, EVT_HANDLE event_handle, bool update_xml, bool resolve, const std::function<bool(std::string_view)>& sid_matcher = none_matcher) { - WindowsEventLogHandler provider{metadata_ptr}; - WindowsEventLogMetadataImpl metadata{provider, event_ptr}; + WindowsEventLogProvider event_log_provider{provider_handle}; + WindowsEventLogMetadataImpl metadata{event_log_provider, event_handle}; MetadataWalker walker(metadata, "", update_xml, resolve, sid_matcher, &utils::OsUtils::userIdToUsername); pugi::xml_document doc; @@ -98,7 +98,7 @@ TEST_CASE("MetadataWalker updates the Sid in the XML if both update_xml and reso SECTION("Resolve nobody") { std::string nobody = readFile("resources/withsids.xml"); - const auto sid_matcher = minifi::processors::cwel::parseSidMatcher(".*Sid"); + const auto sid_matcher = minifi::wel::parseSidMatcher(".*Sid"); REQUIRE(updateXmlMetadata(xml, nullptr, nullptr, true, true, sid_matcher) == formatXml(nobody)); } } @@ -113,7 +113,7 @@ TEST_CASE("MetadataWalker updates the Security/UserId attribute", "[updateXmlMet SECTION("Resolve nobody") { std::string nobody = readFile("resources/resolveduserid.xml"); - const auto sid_matcher = minifi::processors::cwel::parseSidMatcher("(.*Sid)|UserID"); + const auto sid_matcher = minifi::wel::parseSidMatcher("(.*Sid)|UserID"); REQUIRE(updateXmlMetadata(xml, nullptr, nullptr, true, true, sid_matcher) == formatXml(nobody)); } } @@ -135,7 +135,7 @@ TEST_CASE("MetadataWalker will leave a Sid unchanged if it doesn't correspond to REQUIRE(updateXmlMetadata(xml, nullptr, nullptr, false, true) == formatXml(xml)); REQUIRE(updateXmlMetadata(xml, nullptr, nullptr, true, true) == formatXml(xml)); - const auto sid_matcher = minifi::processors::cwel::parseSidMatcher(".*Sid"); + const auto sid_matcher = minifi::wel::parseSidMatcher(".*Sid"); REQUIRE(updateXmlMetadata(xml, nullptr, nullptr, true, true, sid_matcher) == formatXml(xml)); } @@ -157,7 +157,7 @@ void extractMappingsTestHelper(const std::string &file_name, bool update_xml, bool resolve, const std::map<std::string, std::string>& expected_identifiers, - const std::map<METADATA, std::string>& expected_metadata, + const std::map<Metadata, std::string>& expected_metadata, const std::map<std::string, std::string>& expected_field_values) { std::string input_xml = readFile(file_name); REQUIRE(!input_xml.empty()); @@ -165,7 +165,7 @@ void extractMappingsTestHelper(const std::string &file_name, pugi::xml_parse_result result = doc.load_string(input_xml.c_str()); REQUIRE(result); - const auto sid_matcher = minifi::processors::cwel::parseSidMatcher(".*Sid"); + const auto sid_matcher = minifi::wel::parseSidMatcher(".*Sid"); MetadataWalker walker(FakeWindowsEventLogMetadata{}, METADATA_WALKER_TESTS_LOG_NAME, update_xml, resolve, sid_matcher, &utils::OsUtils::userIdToUsername); doc.traverse(walker); @@ -183,12 +183,12 @@ TEST_CASE("MetadataWalker extracts mappings correctly when there is a single Sid const std::map<std::string, std::string> expected_identifiers{{"S-1-0-0", "S-1-0-0"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::EVENT_RECORDID, "2575952"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::EVENT_RECORDID, "2575952"}}; const std::map<std::string, std::string> expected_field_values{}; @@ -206,18 +206,18 @@ TEST_CASE("MetadataWalker extracts mappings correctly when there is a single Sid const std::map<std::string, std::string> expected_identifiers{{"S-1-0-0", "Nobody"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::LOG_NAME, "MetadataWalkerTests"}, - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::OPCODE, "event_data_for_field_4"}, - {METADATA::EVENT_RECORDID, "2575952"}, - {METADATA::EVENT_TYPE, "178"}, - {METADATA::TASK_CATEGORY, "event_data_for_field_3"}, - {METADATA::LEVEL, "event_data_for_field_2"}, - {METADATA::KEYWORDS, "event_data_for_field_5"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::LOG_NAME, "MetadataWalkerTests"}, + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::OPCODE, "event_data_for_field_4"}, + {Metadata::EVENT_RECORDID, "2575952"}, + {Metadata::EVENT_TYPE, "178"}, + {Metadata::TASK_CATEGORY, "event_data_for_field_3"}, + {Metadata::LEVEL, "event_data_for_field_2"}, + {Metadata::KEYWORDS, "event_data_for_field_5"}}; SECTION("update_xml is false => fields are collected into walker.getFieldValues()") { const std::map<std::string, std::string> expected_field_values{ @@ -243,12 +243,12 @@ TEST_CASE("MetadataWalker extracts mappings correctly when there are multiple Si const std::map<std::string, std::string> expected_identifiers{{"S-1-0-0", "S-1-0-0"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::EVENT_RECORDID, "2575952"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::EVENT_RECORDID, "2575952"}}; const std::map<std::string, std::string> expected_field_values{}; @@ -272,18 +272,18 @@ TEST_CASE("MetadataWalker extracts mappings correctly when there are multiple Si {"S-1-0-0", "Nobody"}, {"S-1-1-0", "Everyone"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::LOG_NAME, "MetadataWalkerTests"}, - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::OPCODE, "event_data_for_field_4"}, - {METADATA::EVENT_RECORDID, "2575952"}, - {METADATA::EVENT_TYPE, "178"}, - {METADATA::TASK_CATEGORY, "event_data_for_field_3"}, - {METADATA::LEVEL, "event_data_for_field_2"}, - {METADATA::KEYWORDS, "event_data_for_field_5"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::LOG_NAME, "MetadataWalkerTests"}, + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::OPCODE, "event_data_for_field_4"}, + {Metadata::EVENT_RECORDID, "2575952"}, + {Metadata::EVENT_TYPE, "178"}, + {Metadata::TASK_CATEGORY, "event_data_for_field_3"}, + {Metadata::LEVEL, "event_data_for_field_2"}, + {Metadata::KEYWORDS, "event_data_for_field_5"}}; SECTION("update_xml is false => fields are collected into walker.getFieldValues()") { const std::map<std::string, std::string> expected_field_values{ @@ -309,12 +309,12 @@ TEST_CASE("MetadataWalker extracts mappings correctly when the Sid is unknown an const std::map<std::string, std::string> expected_identifiers{{"S-1-8-6-5-3-0-9", "S-1-8-6-5-3-0-9"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::EVENT_RECORDID, "2575952"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::EVENT_RECORDID, "2575952"}}; const std::map<std::string, std::string> expected_field_values{}; @@ -332,18 +332,18 @@ TEST_CASE("MetadataWalker extracts mappings correctly when the Sid is unknown an const std::map<std::string, std::string> expected_identifiers{{"S-1-8-6-5-3-0-9", "S-1-8-6-5-3-0-9"}}; - using org::apache::nifi::minifi::wel::METADATA; - const std::map<METADATA, std::string> expected_metadata{ - {METADATA::LOG_NAME, "MetadataWalkerTests"}, - {METADATA::SOURCE, "Microsoft-Windows-Security-Auditing"}, - {METADATA::TIME_CREATED, "event_timestamp"}, - {METADATA::EVENTID, "4672"}, - {METADATA::OPCODE, "event_data_for_field_4"}, - {METADATA::EVENT_RECORDID, "2575952"}, - {METADATA::EVENT_TYPE, "178"}, - {METADATA::TASK_CATEGORY, "event_data_for_field_3"}, - {METADATA::LEVEL, "event_data_for_field_2"}, - {METADATA::KEYWORDS, "event_data_for_field_5"}}; + using org::apache::nifi::minifi::wel::Metadata; + const std::map<Metadata, std::string> expected_metadata{ + {Metadata::LOG_NAME, "MetadataWalkerTests"}, + {Metadata::SOURCE, "Microsoft-Windows-Security-Auditing"}, + {Metadata::TIME_CREATED, "event_timestamp"}, + {Metadata::EVENTID, "4672"}, + {Metadata::OPCODE, "event_data_for_field_4"}, + {Metadata::EVENT_RECORDID, "2575952"}, + {Metadata::EVENT_TYPE, "178"}, + {Metadata::TASK_CATEGORY, "event_data_for_field_3"}, + {Metadata::LEVEL, "event_data_for_field_2"}, + {Metadata::KEYWORDS, "event_data_for_field_5"}}; SECTION("update_xml is false => fields are collected into walker.getFieldValues()") { const std::map<std::string, std::string> expected_field_values{ diff --git a/extensions/windows-event-log/tests/StringSplitterTests.cpp b/extensions/windows-event-log/tests/StringSplitterTests.cpp new file mode 100644 index 000000000..907cfb77a --- /dev/null +++ b/extensions/windows-event-log/tests/StringSplitterTests.cpp @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <string_view> +#include <unordered_map> + +#include "unit/Catch.h" +#include "unit/TestBase.h" +#include "unit/TestUtils.h" +#include "wel/StringSplitter.h" + +namespace wel = org::apache::nifi::minifi::wel; + +TEST_CASE("splitCommaSeparatedKeyValuePairs works") { + static constexpr auto test = [](std::string_view input, std::unordered_map<std::string_view, std::string_view> expected) { + std::unordered_map<std::string_view, std::string_view> output; + wel::splitCommaSeparatedKeyValuePairs(input, [&output](std::string_view key, std::string_view value) { + output[key] = value; + }); + CHECK(output == expected); + }; + + test("", {}); + test("foo", {{"foo", ""}}); + test("foo = ", {{"foo", ""}}); + test("foo = bar", {{"foo", "bar"}}); + test("foo == bar", {}); // more than two parts after splitting at '=' + test("foo != bar", {{"foo !", "bar"}}); + test("some long key = a long value", {{"some long key", "a long value"}}); + test("all the leaves \t= brown ,\n and the sky=gray, \r\n I've been for a walk , on a winter's = day", + {{"all the leaves", "brown"}, {"and the sky", "gray"}, {"I've been for a walk", ""}, {"on a winter's", "day"}}); +} diff --git a/extensions/windows-event-log/tests/WindowsErrorTests.cpp b/extensions/windows-event-log/tests/WindowsErrorTests.cpp new file mode 100644 index 000000000..0f4890e02 --- /dev/null +++ b/extensions/windows-event-log/tests/WindowsErrorTests.cpp @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <Windows.h> + +#include <utility> + +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "wel/WindowsError.h" + +namespace wel = org::apache::nifi::minifi::wel; + +TEST_CASE("Windows system errors can be formatted") { + CHECK(fmt::format("{}", wel::WindowsError{ERROR_SUCCESS}) == "error 0x0: The operation completed successfully."); + CHECK(fmt::format("{}", wel::WindowsError{ERROR_INVALID_ADDRESS}) == "error 0x1E7: Attempt to access invalid address."); + CHECK(fmt::format("{}", wel::WindowsError{ERROR_APPCONTAINER_REQUIRED}) == "error 0x109B: This application can only run in the context of an app container."); + CHECK(fmt::format("{}", wel::WindowsError{ERROR_STATE_QUERY_SETTING_FAILED}) == "error 0x3DC2: State Manager failed to query the setting."); + CHECK(fmt::format("{}", wel::WindowsError{16789}) == "error 0x4195: unknown error"); +} diff --git a/extensions/windows-event-log/Bookmark.cpp b/extensions/windows-event-log/wel/Bookmark.cpp similarity index 87% rename from extensions/windows-event-log/Bookmark.cpp rename to extensions/windows-event-log/wel/Bookmark.cpp index a4a08f53e..2adc364c2 100644 --- a/extensions/windows-event-log/Bookmark.cpp +++ b/extensions/windows-event-log/wel/Bookmark.cpp @@ -26,8 +26,9 @@ #include "utils/file/FileUtils.h" #include "utils/OsUtils.h" #include "utils/UnicodeConversion.h" +#include "WindowsError.h" -namespace org::apache::nifi::minifi::processors { +namespace org::apache::nifi::minifi::wel { static const std::string BOOKMARK_KEY = "bookmark"; Bookmark::Bookmark(const wel::EventPath& path, @@ -67,7 +68,7 @@ Bookmark::Bookmark(const wel::EventPath& path, return; } - LOG_LAST_ERROR(EvtCreateBookmark); + logger_->log_error("EvtCreateBookmark failed due to {}", wel::getLastError()); bookmarkXml_.clear(); state_map.erase(BOOKMARK_KEY); @@ -76,18 +77,18 @@ Bookmark::Bookmark(const wel::EventPath& path, hBookmark_ = unique_evt_handle{EvtCreateBookmark(nullptr)}; if (!hBookmark_) { - LOG_LAST_ERROR(EvtCreateBookmark); + logger_->log_error("EvtCreateBookmark failed due to {}", wel::getLastError()); return; } const auto hEventResults = unique_evt_handle{ EvtQuery(nullptr, path.wstr().c_str(), query.c_str(), path.getQueryFlags()) }; if (!hEventResults) { - LOG_LAST_ERROR(EvtQuery); + logger_->log_error("EvtQuery failed due to {}", wel::getLastError()); return; } if (!EvtSeek(hEventResults.get(), 0, nullptr, 0, processOldEvents? EvtSeekRelativeToFirst : EvtSeekRelativeToLast)) { - LOG_LAST_ERROR(EvtSeek); + logger_->log_error("EvtSeek failed due to {}", wel::getLastError()); return; } @@ -95,7 +96,7 @@ Bookmark::Bookmark(const wel::EventPath& path, DWORD dwReturned{}; EVT_HANDLE hEvent{ nullptr }; if (!EvtNext(hEventResults.get(), 1, &hEvent, INFINITE, 0, &dwReturned)) { - LOG_LAST_ERROR(EvtNext); + logger_->log_error("EvtNext failed due to {}", wel::getLastError()); } return unique_evt_handle{ hEvent }; }(); @@ -105,14 +106,14 @@ Bookmark::Bookmark(const wel::EventPath& path, Bookmark::~Bookmark() = default; -Bookmark::operator bool() const noexcept { +bool Bookmark::isValid() const noexcept { return ok_; } EVT_HANDLE Bookmark::getBookmarkHandleFromXML() { hBookmark_ = unique_evt_handle{ EvtCreateBookmark(bookmarkXml_.c_str()) }; if (!hBookmark_) { - LOG_LAST_ERROR(EvtCreateBookmark); + logger_->log_error("EvtCreateBookmark failed due to {}", wel::getLastError()); } return hBookmark_.get(); @@ -139,7 +140,7 @@ bool Bookmark::saveBookmark(EVT_HANDLE event_handle) { nonstd::expected<std::wstring, std::string> Bookmark::getNewBookmarkXml(EVT_HANDLE hEvent) { if (!EvtUpdateBookmark(hBookmark_.get(), hEvent)) { - return nonstd::make_unexpected(fmt::format("EvtUpdateBookmark failed due to {}", utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message())); + return nonstd::make_unexpected(fmt::format("EvtUpdateBookmark failed due to {}", wel::getLastError())); } // Render the bookmark as an XML string that can be persisted. logger_->log_trace("Rendering new bookmark"); @@ -150,15 +151,15 @@ nonstd::expected<std::wstring, std::string> Bookmark::getNewBookmarkXml(EVT_HAND if (event_render_succeeded_without_buffer) return nonstd::make_unexpected("EvtRender failed to determine the required buffer size."); - auto last_error = GetLastError(); - if (last_error != ERROR_INSUFFICIENT_BUFFER) - return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", utils::OsUtils::windowsErrorToErrorCode(last_error).message())); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", wel::getLastError())); + } bufferSize = bufferUsed; std::vector<wchar_t> buf(bufferSize / 2 + 1); if (!EvtRender(nullptr, hBookmark_.get(), EvtRenderBookmark, bufferSize, buf.data(), &bufferUsed, &propertyCount)) { - return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message())); + return nonstd::make_unexpected(fmt::format("EvtRender failed due to {}", wel::getLastError())); } return std::wstring(buf.data()); @@ -203,4 +204,4 @@ bool Bookmark::getBookmarkXmlFromFile(std::wstring& bookmarkXml) { return true; } -} // namespace org::apache::nifi::minifi::processors +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/Bookmark.h b/extensions/windows-event-log/wel/Bookmark.h similarity index 85% rename from extensions/windows-event-log/Bookmark.h rename to extensions/windows-event-log/wel/Bookmark.h index 0b17a2e82..36ffe6eaf 100644 --- a/extensions/windows-event-log/Bookmark.h +++ b/extensions/windows-event-log/wel/Bookmark.h @@ -18,7 +18,7 @@ #pragma once -#include <windows.h> +#include <Windows.h> #include <winevt.h> #include <string> #include <memory> @@ -26,14 +26,12 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" -#include "wel/UniqueEvtHandle.h" #include "core/logging/Logger.h" #include "utils/expected.h" -#include "wel/EventPath.h" +#include "EventPath.h" +#include "UniqueEvtHandle.h" -namespace org::apache::nifi::minifi::processors { - -#define LOG_LAST_ERROR(func) logger_->log_error("!"#func" error {:#x}", GetLastError()) +namespace org::apache::nifi::minifi::wel { class Bookmark { public: @@ -45,8 +43,7 @@ class Bookmark { core::StateManager* state_manager, std::shared_ptr<core::logging::Logger> logger); ~Bookmark(); - explicit operator bool() const noexcept; - + bool isValid() const noexcept; /* non-owning */ EVT_HANDLE getBookmarkHandleFromXML(); nonstd::expected<std::wstring, std::string> getNewBookmarkXml(EVT_HANDLE hEvent); bool saveBookmarkXml(const std::wstring& bookmarkXml); @@ -65,4 +62,4 @@ class Bookmark { std::wstring bookmarkXml_; }; -} // namespace org::apache::nifi::minifi::processors +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/wel/EventPath.h b/extensions/windows-event-log/wel/EventPath.h index dea8d60dc..8a02d5a00 100644 --- a/extensions/windows-event-log/wel/EventPath.h +++ b/extensions/windows-event-log/wel/EventPath.h @@ -18,7 +18,7 @@ #pragma once -#include <windows.h> +#include <Windows.h> #include <winevt.h> #include <string> diff --git a/extensions/windows-event-log/wel/MetadataWalker.cpp b/extensions/windows-event-log/wel/MetadataWalker.cpp index 788f127b4..fee896a27 100644 --- a/extensions/windows-event-log/wel/MetadataWalker.cpp +++ b/extensions/windows-event-log/wel/MetadataWalker.cpp @@ -122,32 +122,42 @@ std::vector<std::string> MetadataWalker::getIdentifiers(const std::string &text) return found_strings; } -std::string MetadataWalker::getMetadata(METADATA metadata) const { - switch (metadata) { - case LOG_NAME: - return log_name_; - case SOURCE: - return getString(metadata_, "Provider"); - case TIME_CREATED: - return windows_event_log_metadata_.getEventTimestamp(); - case EVENTID: - return getString(metadata_, "EventID"); - case EVENT_RECORDID: - return getString(metadata_, "EventRecordID"); - case OPCODE: - return getString(metadata_, "Opcode"); - case TASK_CATEGORY: - return getString(metadata_, "Task"); - case LEVEL: - return getString(metadata_, "Level"); - case KEYWORDS: - return getString(metadata_, "Keywords"); - case EVENT_TYPE: - return std::to_string(windows_event_log_metadata_.getEventTypeIndex()); - case COMPUTER: - return WindowsEventLogMetadata::getComputerName(); +std::string MetadataWalker::getMetadata(Metadata metadata) const { + static constexpr auto get_from_map = [](const std::map<std::string, std::string>& map, const std::string& field) { + auto it = map.find(field); + if (it != std::end(map)) { + return it->second; } - return "N/A"; + return std::string{"N/A"}; + }; + + switch (metadata) { + case Metadata::LOG_NAME: + return log_name_; + case Metadata::SOURCE: + return get_from_map(metadata_, "Provider"); + case Metadata::TIME_CREATED: + return windows_event_log_metadata_.getEventTimestamp(); + case Metadata::EVENTID: + return get_from_map(metadata_, "EventID"); + case Metadata::EVENT_RECORDID: + return get_from_map(metadata_, "EventRecordID"); + case Metadata::OPCODE: + return get_from_map(metadata_, "Opcode"); + case Metadata::TASK_CATEGORY: + return get_from_map(metadata_, "Task"); + case Metadata::LEVEL: + return get_from_map(metadata_, "Level"); + case Metadata::KEYWORDS: + return get_from_map(metadata_, "Keywords"); + case Metadata::EVENT_TYPE: + return std::to_string(windows_event_log_metadata_.getEventTypeIndex()); + case Metadata::COMPUTER: + return WindowsEventLogMetadata::getComputerName(); + case Metadata::USER: + break; + } + return "N/A"; } std::map<std::string, std::string> MetadataWalker::getFieldValues() const { diff --git a/extensions/windows-event-log/wel/MetadataWalker.h b/extensions/windows-event-log/wel/MetadataWalker.h index 9a8080f4d..b9da6c677 100644 --- a/extensions/windows-event-log/wel/MetadataWalker.h +++ b/extensions/windows-event-log/wel/MetadataWalker.h @@ -60,32 +60,13 @@ class MetadataWalker : public pugi::xml_tree_walker { user_id_to_username_fn_(std::move(user_id_to_username_fn)) { } - /** - * Overloaded function to visit a node - * @param node Node that we are visiting. - */ bool for_each(pugi::xml_node &node) override; - [[nodiscard]] std::map<std::string, std::string> getFieldValues() const; - [[nodiscard]] std::map<std::string, std::string> getIdentifiers() const; - - [[nodiscard]] std::string getMetadata(METADATA metadata) const; + [[nodiscard]] std::string getMetadata(Metadata metadata) const; private: static std::vector<std::string> getIdentifiers(const std::string &text); - - static std::string getString(const std::map<std::string, std::string> &map, const std::string &field) { - auto srch = map.find(field); - if (srch != std::end(map)) { - return srch->second; - } - return "N/A"; - } - - /** - * Updates text within the XML representation - */ template<typename Fn> requires std::is_convertible_v<std::invoke_result_t<Fn, std::string>, std::string> void updateText(pugi::xml_node &node, const std::string &field_name, Fn &&fn); diff --git a/extensions/windows-event-log/wel/StringSplitter.h b/extensions/windows-event-log/wel/StringSplitter.h new file mode 100644 index 000000000..5dbf620c2 --- /dev/null +++ b/extensions/windows-event-log/wel/StringSplitter.h @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <cctype> +#include <concepts> +#include <ranges> +#include <string_view> +#include <vector> + +namespace org::apache::nifi::minifi::wel { + +template<std::invocable<std::string_view, std::string_view> Func> +void splitCommaSeparatedKeyValuePairs(std::string_view input, Func output_callback) { + static constexpr auto count_whitespace = [](auto begin, auto end) { + size_t counter = 0; + for (auto it = begin; it != end && std::isspace(static_cast<unsigned char>(*it)); ++it) { ++counter; } + return counter; + }; + for (const auto key_value_str : input | std::views::split(',')) { + const auto key_value_vec = key_value_str + | std::views::split('=') + | std::views::transform([](std::span<const char> term) { + size_t num_trim_left = count_whitespace(term.begin(), term.end()); + if (num_trim_left < term.size()) { + size_t num_trim_right = count_whitespace(term.rbegin(), term.rend()); + return std::string_view{term.data() + num_trim_left, term.size() - (num_trim_left + num_trim_right)}; + } else { + return std::string_view{}; + } + }) + | std::ranges::to<std::vector>(); + if (key_value_vec.size() == 2) { + output_callback(key_value_vec[0], key_value_vec[1]); + } else if (key_value_vec.size() == 1) { + output_callback(key_value_vec[0], ""); + } + } +} + +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/wel/UniqueEvtHandle.h b/extensions/windows-event-log/wel/UniqueEvtHandle.h index 5e19b612d..60574d05b 100644 --- a/extensions/windows-event-log/wel/UniqueEvtHandle.h +++ b/extensions/windows-event-log/wel/UniqueEvtHandle.h @@ -17,15 +17,11 @@ #pragma once -#include <windows.h> +#include <Windows.h> #include <memory> -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace wel { +namespace org::apache::nifi::minifi::wel { struct unique_evt_handle_deleter { void operator()(EVT_HANDLE event) const noexcept { EvtClose(event); } @@ -33,9 +29,4 @@ struct unique_evt_handle_deleter { using unique_evt_handle = std::unique_ptr<std::remove_pointer<EVT_HANDLE>::type, unique_evt_handle_deleter>; -} /* namespace wel */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ - +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/wel/UniqueEvtHandle.h b/extensions/windows-event-log/wel/WindowsError.h similarity index 58% copy from extensions/windows-event-log/wel/UniqueEvtHandle.h copy to extensions/windows-event-log/wel/WindowsError.h index 5e19b612d..2b060ef20 100644 --- a/extensions/windows-event-log/wel/UniqueEvtHandle.h +++ b/extensions/windows-event-log/wel/WindowsError.h @@ -17,25 +17,26 @@ #pragma once -#include <windows.h> +#include <Windows.h> -#include <memory> +#include <system_error> -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace wel { +#include "fmt/format.h" -struct unique_evt_handle_deleter { - void operator()(EVT_HANDLE event) const noexcept { EvtClose(event); } +namespace org::apache::nifi::minifi::wel { + +struct WindowsError { + DWORD error_code; }; -using unique_evt_handle = std::unique_ptr<std::remove_pointer<EVT_HANDLE>::type, unique_evt_handle_deleter>; +inline WindowsError getLastError() { + return {GetLastError()}; +} -} /* namespace wel */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ +// this is called by fmt::format; see https://fmt.dev/11.1/api/#formatting-user-defined-types +inline std::string format_as(WindowsError windows_error) { + std::error_code error_code(windows_error.error_code, std::system_category()); + return fmt::format("error 0x{:X}: {}", error_code.value(), error_code.message()); +} +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/wel/WindowsEventLog.cpp b/extensions/windows-event-log/wel/WindowsEventLog.cpp index 585333e99..ad554e615 100644 --- a/extensions/windows-event-log/wel/WindowsEventLog.cpp +++ b/extensions/windows-event-log/wel/WindowsEventLog.cpp @@ -67,7 +67,7 @@ void WindowsEventLogMetadataImpl::renderMetadata() { if (!context) return; - if (!EvtRender(context.get(), event_ptr_, EvtRenderEventValues, dwBufferSize, rendered_values.get(), &dwBufferUsed, &dwPropertyCount)) { + if (!EvtRender(context.get(), event_handle_, EvtRenderEventValues, dwBufferSize, rendered_values.get(), &dwBufferUsed, &dwPropertyCount)) { if (ERROR_INSUFFICIENT_BUFFER != (status = GetLastError())) { return; } @@ -78,7 +78,7 @@ void WindowsEventLogMetadataImpl::renderMetadata() { return; } - EvtRender(context.get(), event_ptr_, EvtRenderEventValues, dwBufferSize, rendered_values.get(), &dwBufferUsed, &dwPropertyCount); + EvtRender(context.get(), event_handle_, EvtRenderEventValues, dwBufferSize, rendered_values.get(), &dwBufferUsed, &dwPropertyCount); if (ERROR_SUCCESS != (status = GetLastError())) { return; } @@ -121,29 +121,29 @@ void WindowsEventLogMetadataImpl::renderMetadata() { } std::string WindowsEventLogMetadataImpl::getEventData(EVT_FORMAT_MESSAGE_FLAGS field, const std::string& key) const { - return metadata_ptr_.getEventData(field, key, event_ptr_); + return event_log_provider_.getEventData(field, key, event_handle_); } -std::string WindowsEventLogHandler::getEventData(EVT_FORMAT_MESSAGE_FLAGS field, const std::string& key, EVT_HANDLE event_ptr) const { +std::string WindowsEventLogProvider::getEventData(EVT_FORMAT_MESSAGE_FLAGS field, const std::string& key, EVT_HANDLE event_handle) const { auto cached_value = event_data_cache_.get(field, key); if (cached_value) { return *cached_value; } - auto new_value = getEventDataImpl(field, event_ptr); + auto new_value = getEventDataImpl(field, event_handle); event_data_cache_.set(field, key, new_value); return new_value; } -std::string WindowsEventLogHandler::getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS field, EVT_HANDLE event_ptr) const { +std::string WindowsEventLogProvider::getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS field, EVT_HANDLE event_handle) const { WCHAR stack_buffer[4096]; DWORD num_chars_in_buffer = sizeof(stack_buffer) / sizeof(stack_buffer[0]); using Deleter = utils::StackAwareDeleter<WCHAR, utils::FreeDeleter>; std::unique_ptr<WCHAR, Deleter> buffer{stack_buffer, Deleter{stack_buffer}}; DWORD num_chars_used = 0; - if (!metadata_provider_ || !event_ptr) { + if (!provider_handle_ || !event_handle) { return {}; } - if (!EvtFormatMessage(metadata_provider_.get(), event_ptr, 0, 0, nullptr, field, num_chars_in_buffer, buffer.get(), &num_chars_used)) { + if (!EvtFormatMessage(provider_handle_.get(), event_handle, 0, 0, nullptr, field, num_chars_in_buffer, buffer.get(), &num_chars_used)) { auto last_error = GetLastError(); if (ERROR_INSUFFICIENT_BUFFER == last_error) { num_chars_in_buffer = num_chars_used; @@ -153,7 +153,7 @@ std::string WindowsEventLogHandler::getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS fi return {}; } - EvtFormatMessage(metadata_provider_.get(), event_ptr, 0, 0, nullptr, field, num_chars_in_buffer, buffer.get(), &num_chars_used); + EvtFormatMessage(provider_handle_.get(), event_handle, 0, 0, nullptr, field, num_chars_in_buffer, buffer.get(), &num_chars_used); } } @@ -167,7 +167,7 @@ std::string WindowsEventLogHandler::getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS fi return utils::to_string(std::wstring{buffer.get()}); } -nonstd::expected<std::string, std::error_code> WindowsEventLogHandler::getEventMessage(EVT_HANDLE eventHandle) const { +nonstd::expected<std::string, std::error_code> WindowsEventLogProvider::getEventMessage(EVT_HANDLE event_handle) const { std::string returnValue; WCHAR stack_buffer[4096]; DWORD num_chars_in_buffer = sizeof(stack_buffer) / sizeof(stack_buffer[0]); @@ -175,7 +175,7 @@ nonstd::expected<std::string, std::error_code> WindowsEventLogHandler::getEventM std::unique_ptr<WCHAR, Deleter> buffer{stack_buffer, Deleter{stack_buffer}}; DWORD num_chars_used = 0; - bool evt_format_succeeded = EvtFormatMessage(metadata_provider_.get(), eventHandle, 0, 0, nullptr, EvtFormatMessageEvent, num_chars_in_buffer, buffer.get(), &num_chars_used); + bool evt_format_succeeded = EvtFormatMessage(provider_handle_.get(), event_handle, 0, 0, nullptr, EvtFormatMessageEvent, num_chars_in_buffer, buffer.get(), &num_chars_used); if (evt_format_succeeded) return utils::to_string(std::wstring{buffer.get()}); @@ -188,7 +188,7 @@ nonstd::expected<std::string, std::error_code> WindowsEventLogHandler::getEventM buffer.reset((LPWSTR) malloc(num_chars_in_buffer * sizeof(WCHAR))); if (!buffer) return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(ERROR_OUTOFMEMORY)); - if (EvtFormatMessage(metadata_provider_.get(), eventHandle, 0, 0, nullptr, + if (EvtFormatMessage(provider_handle_.get(), event_handle, 0, 0, nullptr, EvtFormatMessageEvent, num_chars_in_buffer, buffer.get(), &num_chars_used)) return utils::to_string(std::wstring{buffer.get()}); @@ -196,7 +196,7 @@ nonstd::expected<std::string, std::error_code> WindowsEventLogHandler::getEventM } namespace { -size_t findLongestHeaderNameSize(const METADATA_NAMES& header_names, const size_t minimum_size) { +size_t findLongestHeaderNameSize(const HeaderNames& header_names, const size_t minimum_size) { size_t max = minimum_size; for (const auto& option : header_names) { max = (std::max(max, option.second.size())); @@ -205,7 +205,7 @@ size_t findLongestHeaderNameSize(const METADATA_NAMES& header_names, const size_ } } // namespace -WindowsEventLogHeader::WindowsEventLogHeader(const METADATA_NAMES& header_names, const std::optional<std::string>& custom_delimiter, const size_t minimum_size) +WindowsEventLogHeader::WindowsEventLogHeader(const HeaderNames& header_names, const std::optional<std::string>& custom_delimiter, const size_t minimum_size) : header_names_(header_names), custom_delimiter_(custom_delimiter), longest_header_name_(findLongestHeaderNameSize(header_names, minimum_size)) { diff --git a/extensions/windows-event-log/wel/WindowsEventLog.h b/extensions/windows-event-log/wel/WindowsEventLog.h index 7bfae03d2..b73c54573 100644 --- a/extensions/windows-event-log/wel/WindowsEventLog.h +++ b/extensions/windows-event-log/wel/WindowsEventLog.h @@ -43,7 +43,7 @@ namespace org::apache::nifi::minifi::wel { -enum METADATA { +enum class Metadata { LOG_NAME, SOURCE, TIME_CREATED, @@ -56,9 +56,7 @@ enum METADATA { KEYWORDS, USER, COMPUTER, - UNKNOWN }; -using METADATA_NAMES = std::vector<std::pair<METADATA, std::string>>; class EventDataCache { public: @@ -89,22 +87,18 @@ class EventDataCache { std::unordered_map<CacheKey, CacheItem, CacheKeyHash> cache_; }; -class WindowsEventLogHandler { +class WindowsEventLogProvider { public: - WindowsEventLogHandler() : metadata_provider_(nullptr) { - } - - explicit WindowsEventLogHandler(EVT_HANDLE metadataProvider) : metadata_provider_(metadataProvider) { - } + WindowsEventLogProvider() : provider_handle_(nullptr) {} + explicit WindowsEventLogProvider(EVT_HANDLE provider_handle) : provider_handle_(provider_handle) {} - nonstd::expected<std::string, std::error_code> getEventMessage(EVT_HANDLE eventHandle) const; - - [[nodiscard]] std::string getEventData(EVT_FORMAT_MESSAGE_FLAGS field, const std::string& key, EVT_HANDLE event_ptr) const; + nonstd::expected<std::string, std::error_code> getEventMessage(EVT_HANDLE event_handle) const; + [[nodiscard]] std::string getEventData(EVT_FORMAT_MESSAGE_FLAGS field, const std::string& key, EVT_HANDLE event_handle) const; private: - [[nodiscard]] std::string getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS field, EVT_HANDLE event_ptr) const; + [[nodiscard]] std::string getEventDataImpl(EVT_FORMAT_MESSAGE_FLAGS field, EVT_HANDLE event_handle) const; - unique_evt_handle metadata_provider_; + unique_evt_handle provider_handle_; mutable EventDataCache event_data_cache_; }; @@ -115,49 +109,6 @@ class WindowsEventLogMetadata { [[nodiscard]] virtual std::string getEventTimestamp() const = 0; virtual short getEventTypeIndex() const = 0; // NOLINT short comes from WINDOWS API - static std::string getMetadataString(METADATA val) { - static std::map<METADATA, std::string> map = { - {LOG_NAME, "LOG_NAME"}, - {SOURCE, "SOURCE"}, - {TIME_CREATED, "TIME_CREATED"}, - {EVENTID, "EVENTID"}, - {OPCODE, "OPCODE"}, - {EVENT_RECORDID, "EVENT_RECORDID"}, - {EVENT_TYPE, "EVENT_TYPE"}, - {TASK_CATEGORY, "TASK_CATEGORY"}, - {LEVEL, "LEVEL"}, - {KEYWORDS, "KEYWORDS"}, - {USER, "USER"}, - {COMPUTER, "COMPUTER"} - }; - - return map[val]; - } - - static METADATA getMetadataFromString(const std::string& val) { - static std::map<std::string, METADATA> map = { - {"LOG_NAME", LOG_NAME}, - {"SOURCE", SOURCE}, - {"TIME_CREATED", TIME_CREATED}, - {"EVENTID", EVENTID}, - {"OPCODE", OPCODE}, - {"EVENT_RECORDID", EVENT_RECORDID}, - {"TASK_CATEGORY", TASK_CATEGORY}, - {"EVENT_TYPE", EVENT_TYPE}, - {"LEVEL", LEVEL}, - {"KEYWORDS", KEYWORDS}, - {"USER", USER}, - {"COMPUTER", COMPUTER} - }; - - auto enumVal = map.find(val); - if (enumVal != std::end(map)) { - return enumVal->second; - } else { - return METADATA::UNKNOWN; - } - } - static std::string getComputerName() { static std::string computer_name; if (computer_name.empty()) { @@ -175,7 +126,7 @@ class WindowsEventLogMetadata { class WindowsEventLogMetadataImpl : public WindowsEventLogMetadata { public: - WindowsEventLogMetadataImpl(const WindowsEventLogHandler& metadataProvider, EVT_HANDLE event_ptr) : metadata_ptr_(metadataProvider), event_ptr_(event_ptr) { + WindowsEventLogMetadataImpl(const WindowsEventLogProvider& event_log_provider, EVT_HANDLE event_handle) : event_log_provider_(event_log_provider), event_handle_(event_handle) { renderMetadata(); } @@ -191,13 +142,15 @@ class WindowsEventLogMetadataImpl : public WindowsEventLogMetadata { std::string event_type_; short event_type_index_ = 0; // NOLINT short comes from WINDOWS API std::string event_timestamp_str_; - EVT_HANDLE event_ptr_; - const WindowsEventLogHandler& metadata_ptr_; + EVT_HANDLE event_handle_; + const WindowsEventLogProvider& event_log_provider_; }; +using HeaderNames = std::vector<std::pair<Metadata, std::string>>; + class WindowsEventLogHeader { public: - explicit WindowsEventLogHeader(const METADATA_NAMES& header_names, const std::optional<std::string>& custom_delimiter, size_t minimum_size); + explicit WindowsEventLogHeader(const HeaderNames& header_names, const std::optional<std::string>& custom_delimiter, size_t minimum_size); template<typename MetadataCollection> std::string getEventHeader(const MetadataCollection& metadata_collection) const; @@ -206,7 +159,7 @@ class WindowsEventLogHeader { private: [[nodiscard]] std::string createDefaultDelimiter(size_t length) const; - const METADATA_NAMES& header_names_; + const HeaderNames& header_names_; const std::optional<std::string>& custom_delimiter_; const size_t longest_header_name_; }; diff --git a/libminifi/test/libtest/unit/Catch.h b/libminifi/test/libtest/unit/Catch.h index 6ce6fa833..3a6ff3bce 100644 --- a/libminifi/test/libtest/unit/Catch.h +++ b/libminifi/test/libtest/unit/Catch.h @@ -22,6 +22,7 @@ #include <chrono> #include "fmt/format.h" #include "catch2/catch_test_macros.hpp" +#include "catch2/catch_tostring.hpp" #include "catch2/matchers/catch_matchers.hpp" namespace Catch { diff --git a/libminifi/test/libtest/unit/TestUtils.h b/libminifi/test/libtest/unit/TestUtils.h index 13d7b7d24..89954e3a4 100644 --- a/libminifi/test/libtest/unit/TestUtils.h +++ b/libminifi/test/libtest/unit/TestUtils.h @@ -32,7 +32,6 @@ #include "utils/TimeUtil.h" #include "TestBase.h" -#include "catch2/catch_tostring.hpp" #include "fmt/format.h" #include "rapidjson/document.h" #include "asio.hpp" @@ -244,4 +243,13 @@ struct StringMaker<minifi::state::response::ValueNode> { return fmt::format(R"("{}")", value_node.to_string()); } }; + +template <> +struct StringMaker<std::unordered_map<std::string_view, std::string_view>> { + static std::string convert(const std::unordered_map<std::string_view, std::string_view>& map) { + return "{" + utils::string::join(", ", map, [](const auto& kv) { + return fmt::format(R"("{}" => "{}")", kv.first, kv.second); + }) + "}"; + } +}; } // namespace Catch
