This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 078fd1c87463ea227764f03b98b6bdc539074dfe Author: Adam Debreceni <[email protected]> AuthorDate: Mon Apr 17 10:50:11 2023 +0200 MINIFICPP-2081 - Make it possible to read Windows events from a log file Signed-off-by: Ferenc Gerlits <[email protected]> This closes #1564 --- PROCESSORS.md | 2 +- extensions/windows-event-log/Bookmark.cpp | 4 +- extensions/windows-event-log/Bookmark.h | 3 +- .../windows-event-log/ConsumeWindowsEventLog.cpp | 22 +++++--- .../windows-event-log/ConsumeWindowsEventLog.h | 6 +- .../windows-event-log/tests/BookmarkTests.cpp | 30 +++++++++- extensions/windows-event-log/tests/CWELTestUtils.h | 10 ++++ .../tests/ConsumeWindowsEventLogTests.cpp | 41 ++++++++++++++ extensions/windows-event-log/wel/EventPath.cpp | 39 +++++++++++++ extensions/windows-event-log/wel/EventPath.h | 64 ++++++++++++++++++++++ 10 files changed, 204 insertions(+), 17 deletions(-) diff --git a/PROCESSORS.md b/PROCESSORS.md index 1fe86736b..c3ebccb10 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -460,7 +460,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description [...] |------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...] -| **Channel** | System | | The Windows Event Log Channel to listen to.<br/>**Supports Expression Language: true** [...] +| **Channel** | System | | The Windows Event Log Channel to listen to. In order to process logs from a log file use the format 'SavedLog:\<file path\>'. <br/>**Supports Expression Language: true** [...] | **Query** | * | | XPath Query to filter events. (See https://msdn.microsoft.com/en-us/library/windows/desktop/dd996910(v=vs.85).aspx for examples.)<br/>**Supports Expression Language: true** [...] | **Max Buffer Size** | 1 MB | | The individual Event Log XMLs are rendered to a buffer. This specifies the maximum size in bytes that the buffer will be allowed to grow to. (Limiting the maximum size of an individual Event XM [...] | **Inactive Duration To Reconnect** | 10 min | | If no new event logs are processed for the specified time period, this processor will try reconnecting to recover from a state where any further messages cannot be consumed. Such situation can [...] diff --git a/extensions/windows-event-log/Bookmark.cpp b/extensions/windows-event-log/Bookmark.cpp index 74ea2238e..dcdaadde5 100644 --- a/extensions/windows-event-log/Bookmark.cpp +++ b/extensions/windows-event-log/Bookmark.cpp @@ -30,7 +30,7 @@ namespace org::apache::nifi::minifi::processors { static const std::string BOOKMARK_KEY = "bookmark"; -Bookmark::Bookmark(const std::wstring& channel, +Bookmark::Bookmark(const wel::EventPath& path, const std::wstring& query, const std::filesystem::path& bookmarkRootDir, const utils::Identifier& uuid, @@ -80,7 +80,7 @@ Bookmark::Bookmark(const std::wstring& channel, return; } - const auto hEventResults = unique_evt_handle{ EvtQuery(nullptr, channel.c_str(), query.c_str(), EvtQueryChannelPath) }; + const auto hEventResults = unique_evt_handle{ EvtQuery(nullptr, path.wstr().c_str(), query.c_str(), path.getQueryFlags()) }; if (!hEventResults) { LOG_LAST_ERROR(EvtQuery); return; diff --git a/extensions/windows-event-log/Bookmark.h b/extensions/windows-event-log/Bookmark.h index c2f07fb81..0d4d6bf4d 100644 --- a/extensions/windows-event-log/Bookmark.h +++ b/extensions/windows-event-log/Bookmark.h @@ -29,6 +29,7 @@ #include "wel/UniqueEvtHandle.h" #include "logging/Logger.h" #include "utils/expected.h" +#include "wel/EventPath.h" namespace org::apache::nifi::minifi::processors { @@ -36,7 +37,7 @@ namespace org::apache::nifi::minifi::processors { class Bookmark { public: - Bookmark(const std::wstring& channel, + Bookmark(const wel::EventPath& path, const std::wstring& query, const std::filesystem::path& bookmarkRootDir, const utils::Identifier& uuid, diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index d37526c6c..c8cd6b456 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -66,7 +66,7 @@ const core::Property ConsumeWindowsEventLog::Channel( core::PropertyBuilder::createProperty("Channel")-> isRequired(true)-> withDefaultValue("System")-> - withDescription("The Windows Event Log Channel to listen to.")-> + withDescription("The Windows Event Log Channel to listen to. In order to process logs from a log file use the format 'SavedLog:<file path>'.")-> supportsExpressionLanguage(true)-> build()); @@ -282,12 +282,16 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptr<core::ProcessConte } } - context->getProperty(Channel.getName(), channel_); - wstrChannel_ = std::wstring(channel_.begin(), channel_.end()); + path_ = wel::EventPath{context->getProperty(Channel).value()}; + if (path_.kind() == wel::EventPath::Kind::FILE) { + logger_->log_debug("Using saved log file as log source"); + } else { + logger_->log_debug("Using channel as log source"); + } std::string query; context->getProperty(Query.getName(), query); - wstrQuery_ = std::wstring(query.begin(), query.end()); + wstr_query_ = std::wstring(query.begin(), query.end()); bool processOldEvents{}; context->getProperty(ProcessOldEvents.getName(), processOldEvents); @@ -299,7 +303,7 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptr<core::ProcessConte logger_->log_error("State Directory is empty"); throw Exception(PROCESS_SCHEDULE_EXCEPTION, "State Directory is empty"); } - bookmark_ = std::make_unique<Bookmark>(wstrChannel_, wstrQuery_, bookmarkDir, getUUID(), processOldEvents, state_manager_, logger_); + bookmark_ = std::make_unique<Bookmark>(path_, wstr_query_, bookmarkDir, getUUID(), processOldEvents, state_manager_, logger_); if (!*bookmark_) { bookmark_.reset(); throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Bookmark is empty"); @@ -312,7 +316,7 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptr<core::ProcessConte context->getProperty(CacheSidLookups.getName(), cache_sid_lookups_); logger_->log_debug("ConsumeWindowsEventLog: will%s cache SID to name lookups", cache_sid_lookups_ ? "" : " not"); - provenanceUri_ = "winlog://" + computerName_ + "/" + channel_ + "?" + query; + provenanceUri_ = "winlog://" + computerName_ + "/" + path_.str() + "?" + query; logger_->log_trace("Successfully configured CWEL"); } @@ -396,14 +400,14 @@ void ConsumeWindowsEventLog::onTrigger(const std::shared_ptr<core::ProcessContex logger_->log_debug("processed %zu Events in %" PRId64 " ms", processed_event_count, time_diff()); }); - wel::unique_evt_handle event_query_results{EvtQuery(nullptr, wstrChannel_.c_str(), wstrQuery_.c_str(), EvtQueryChannelPath)}; + wel::unique_evt_handle event_query_results{EvtQuery(nullptr, path_.wstr().c_str(), wstr_query_.c_str(), path_.getQueryFlags())}; if (!event_query_results) { LOG_LAST_ERROR(EvtQuery); context->yield(); return; } - logger_->log_trace("Retrieved results in Channel: %ls with Query: %ls", wstrChannel_.c_str(), wstrQuery_.c_str()); + logger_->log_trace("Retrieved results in Channel: %ls with Query: %ls", path_.wstr().c_str(), wstr_query_.c_str()); auto bookmark_handle = bookmark_->getBookmarkHandleFromXML(); if (!bookmark_handle) { @@ -581,7 +585,7 @@ nonstd::expected<EventRender, std::string> ConsumeWindowsEventLog::createEventRe // this is a well known path. std::string provider_name = doc.child("Event").child("System").child("Provider").attribute("Name").value(); wel::WindowsEventLogMetadataImpl metadata{getEventLogHandler(provider_name).getMetadata(), hEvent}; - wel::MetadataWalker walker{metadata, channel_, !resolve_as_attributes_, apply_identifier_function_, regex_ ? &*regex_ : nullptr, userIdToUsernameFunction()}; + wel::MetadataWalker walker{metadata, path_.str(), !resolve_as_attributes_, apply_identifier_function_, regex_ ? &*regex_ : nullptr, userIdToUsernameFunction()}; // resolve the event metadata doc.traverse(walker); diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index 85c3db70f..7fd8932ed 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -39,6 +39,7 @@ #include "core/ProcessSession.h" #include "utils/OsUtils.h" #include "wel/WindowsEventLog.h" +#include "wel/EventPath.h" #include "FlowFileRecord.h" #include "concurrentqueue.h" #include "pugixml.hpp" @@ -146,9 +147,8 @@ class ConsumeWindowsEventLog : public core::Processor { core::StateManager* state_manager_{nullptr}; wel::METADATA_NAMES header_names_; std::optional<std::string> header_delimiter_; - std::string channel_; - std::wstring wstrChannel_; - std::wstring wstrQuery_; + wel::EventPath path_; + std::wstring wstr_query_; std::optional<utils::Regex> regex_; bool resolve_as_attributes_{false}; bool apply_identifier_function_{false}; diff --git a/extensions/windows-event-log/tests/BookmarkTests.cpp b/extensions/windows-event-log/tests/BookmarkTests.cpp index 139455d16..495f23fa3 100644 --- a/extensions/windows-event-log/tests/BookmarkTests.cpp +++ b/extensions/windows-event-log/tests/BookmarkTests.cpp @@ -24,6 +24,7 @@ #include "Catch.h" #include "utils/gsl.h" #include "wel/UniqueEvtHandle.h" +#include "CWELTestUtils.h" using Bookmark = org::apache::nifi::minifi::processors::Bookmark; using unique_evt_handle = org::apache::nifi::minifi::wel::unique_evt_handle; @@ -42,7 +43,8 @@ std::unique_ptr<Bookmark> createBookmark(TestPlan &test_plan, const utils::Identifier &uuid, core::StateManager* state_manager) { const auto logger = test_plan.getLogger(); - return std::make_unique<Bookmark>(channel, L"*", "", uuid, false, state_manager, logger); + + return std::make_unique<Bookmark>(minifi::wel::EventPath{channel}, L"*", "", uuid, false, state_manager, logger); } void reportEvent(const std::wstring& channel, const char* message) { @@ -114,6 +116,32 @@ TEST_CASE("Bookmark constructor works", "[create]") { REQUIRE(std::regex_match(bookmarkAsXml(bookmark), pattern)); } +TEST_CASE("Bookmark constructor works for log file path", "[create]") { + TestController test_controller; + std::filesystem::path log_file = test_controller.createTempDirectory() / "events.evtx"; + std::shared_ptr<TestPlan> test_plan = test_controller.createPlan(); + LogTestController::getInstance().setTrace<TestPlan>(); + + reportEvent(APPLICATION_CHANNEL, "Publish an event to make sure the event log is not empty"); + generateLogFile(APPLICATION_CHANNEL, log_file); + + + const utils::Identifier uuid = IdGenerator::getIdGenerator()->generate(); + auto state_manager = test_plan->getStateStorage()->getStateManager(uuid); + std::unique_ptr<Bookmark> bookmark = createBookmark(*test_plan, L"SavedLog:" + log_file.wstring(), uuid, state_manager.get()); + REQUIRE(bookmark); + REQUIRE(*bookmark); + + std::string log_file_str = log_file.string(); + utils::StringUtils::replaceAll(log_file_str, "\\", "\\\\"); + + std::wstring pattern{L"<BookmarkList Direction='backward'>\r\n"}; + pattern += L" <Bookmark Channel='" + std::wstring(log_file_str.begin(), log_file_str.end()) + L"' RecordId='\\d+' IsCurrent='true'/>\r\n"; + pattern += L"</BookmarkList>"; + + REQUIRE(std::regex_match(bookmarkAsXml(bookmark), std::wregex{pattern})); +} + TEST_CASE("Bookmark is restored from the state", "[create][state]") { TestController test_controller; std::shared_ptr<TestPlan> test_plan = test_controller.createPlan(); diff --git a/extensions/windows-event-log/tests/CWELTestUtils.h b/extensions/windows-event-log/tests/CWELTestUtils.h index e576c5b2c..d9dd0c865 100644 --- a/extensions/windows-event-log/tests/CWELTestUtils.h +++ b/extensions/windows-event-log/tests/CWELTestUtils.h @@ -30,6 +30,7 @@ #include "Catch.h" #include "utils/TestUtils.h" #include "utils/file/FileUtils.h" +#include "utils/gsl.h" core::Relationship Success{"success", "Everything is fine"}; @@ -94,3 +95,12 @@ class OutputFormatTestController : public TestController { std::string output_format_; std::optional<std::string> json_format_; }; + +void generateLogFile(const std::wstring& channel, const std::filesystem::path& path) { + HANDLE event_log = OpenEventLog(NULL, std::string(channel.begin(), channel.end()).c_str()); + auto guard = gsl::finally([&] {CloseEventLog(event_log);}); + + if (!EvtExportLog(NULL, channel.c_str(), L"*", path.wstring().c_str(), EvtExportLogChannelPath)) { + throw std::system_error{gsl::narrow<int>(GetLastError()), std::system_category(), "Failed to export logs"}; + } +} diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index d0e4d39d7..3b174f2dc 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -65,6 +65,28 @@ class SimpleFormatTestController : public OutputFormatTestController { } }; +class LogFileTestController : public OutputFormatTestController { + public: + LogFileTestController(): OutputFormatTestController("", "*", "JSON", "Simple") { + log_file_ = createTempDirectory() / "cwel-events.evtx"; + channel_ = "SavedLog:" + log_file_.string(); + } + + protected: + void dispatchBookmarkEvent() override { + reportEvent(APPLICATION_CHANNEL, "Event zero from file: this is in the past"); + generateLogFile(std::wstring{APPLICATION_CHANNEL.begin(), APPLICATION_CHANNEL.end()}, log_file_); + } + void dispatchCollectedEvent() override { + reportEvent(APPLICATION_CHANNEL, "Event one from file"); + std::filesystem::remove(log_file_); + generateLogFile(std::wstring{APPLICATION_CHANNEL.begin(), APPLICATION_CHANNEL.end()}, log_file_); + reportEvent(APPLICATION_CHANNEL, "Event not processed after log file export"); + } + + std::filesystem::path log_file_; +}; + } // namespace TEST_CASE("ConsumeWindowsEventLog constructor works", "[create]") { @@ -527,4 +549,23 @@ TEST_CASE("ConsumeWindowsEventLog Simple JSON works with UserData", "[cwel][json } } +TEST_CASE("ConsumeWindowsEventLog can process events from a log file", "[cwel][logfile]") { + std::string event = LogFileTestController{}.run(); + utils::verifyJSON(event, R"json( + { + "System": { + "Provider": { + "Name": "Application" + }, + "Channel": "Application" + }, + "EventData": [{ + "Type": "Data", + "Content": "Event one from file", + "Name": "" + }] + } + )json"); +} + } // namespace org::apache::nifi::minifi::test diff --git a/extensions/windows-event-log/wel/EventPath.cpp b/extensions/windows-event-log/wel/EventPath.cpp new file mode 100644 index 000000000..adc4d9ad1 --- /dev/null +++ b/extensions/windows-event-log/wel/EventPath.cpp @@ -0,0 +1,39 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "EventPath.h" +#include <utility> +#include "utils/StringUtils.h" + +namespace org::apache::nifi::minifi::wel { + +EventPath::EventPath(std::wstring wstr) : EventPath(std::string(wstr.begin(), wstr.end())) {} + +EventPath::EventPath(std::string str) { + constexpr std::string_view saved_log_prefix = "SavedLog:"; + if (utils::StringUtils::startsWith(str, saved_log_prefix)) { + str_ = str.substr(saved_log_prefix.size()); + kind_ = Kind::FILE; + } else { + str_ = std::move(str); + kind_ = Kind::CHANNEL; + } + wstr_ = std::wstring(str_.begin(), str_.end()); +} + +} // namespace org::apache::nifi::minifi::wel diff --git a/extensions/windows-event-log/wel/EventPath.h b/extensions/windows-event-log/wel/EventPath.h new file mode 100644 index 000000000..5b4c295d6 --- /dev/null +++ b/extensions/windows-event-log/wel/EventPath.h @@ -0,0 +1,64 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <windows.h> +#include <winevt.h> +#include <string> + +namespace org::apache::nifi::minifi::wel { + +class EventPath { + public: + enum class Kind { + CHANNEL, + FILE + }; + + EventPath() = default; + explicit EventPath(std::wstring wstr); + explicit EventPath(std::string str); + + constexpr const std::wstring& wstr() const noexcept { + return wstr_; + } + + constexpr const std::string& str() const noexcept { + return str_; + } + + constexpr EventPath::Kind kind() const noexcept { + return kind_; + } + + constexpr EVT_QUERY_FLAGS getQueryFlags() const noexcept { + switch (kind_) { + case Kind::CHANNEL: return EvtQueryChannelPath; + case Kind::FILE: return EvtQueryFilePath; + } + } + + private: + std::string str_; + std::wstring wstr_; + + Kind kind_{Kind::CHANNEL}; +}; + +} // namespace org::apache::nifi::minifi::wel
