adamdebreceni commented on code in PR #1367: URL: https://github.com/apache/nifi-minifi-cpp/pull/1367#discussion_r940173993
########## libminifi/src/core/logging/alert/AlertSink.cpp: ########## @@ -0,0 +1,267 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/logging/alert/AlertSink.h" +#include "core/TypedValues.h" +#include "core/ClassLoader.h" +#include "utils/HTTPClient.h" +#include "utils/Hash.h" +#include "core/logging/Utils.h" + +#include "rapidjson/rapidjson.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace org::apache::nifi::minifi::core::logging { + +AlertSink::AlertSink(Config config, std::shared_ptr<Logger> logger) + : config_(std::move(config)), + buffer_(config_.buffer_limit, config_.batch_size), + logger_(std::move(logger)) { + set_level(config_.level); + live_logs_.setLifetime(config_.rate_limit); + next_flush_ = clock_->timeSinceEpoch() + config_.flush_period; + flush_thread_ = std::thread([this] {run();}); +} + +std::shared_ptr<AlertSink> AlertSink::create(const std::string& prop_name_prefix, const std::shared_ptr<LoggerProperties>& logger_properties, std::shared_ptr<Logger> logger) { + Config config; + + if (auto url = logger_properties->getString(prop_name_prefix + ".url")) { + config.url = url.value(); + } else { + logger->log_info("Missing '%s.url' value, network logging won't be available", prop_name_prefix); + return {}; + } + + if (auto filter_str = logger_properties->getString(prop_name_prefix + ".filter")) { + try { + config.filter = filter_str.value(); + } catch (const std::regex_error& err) { + logger->log_error("Invalid '%s.filter' value, network logging won't be available: %s", prop_name_prefix, err.what()); + return {}; + } + } else { + logger->log_error("Missing '%s.filter' value, network logging won't be available", prop_name_prefix); + return {}; + } + + auto readPropertyOr = [&] (auto suffix, auto parser, auto fallback) { + if (auto prop_str = logger_properties->getString(prop_name_prefix + suffix)) { + if (auto prop_val = parser(prop_str.value())) { + return prop_val.value(); + } + logger->log_error("Invalid '%s' value, using default '%s'", prop_name_prefix + suffix, fallback); + } else { + logger->log_info("Missing '%s' value, using default '%s'", prop_name_prefix + suffix, fallback); + } + return parser(fallback).value(); + }; + + auto datasize_parser = [] (const std::string& str) -> std::optional<int> { + int val; + if (DataSizeValue::StringToInt(str, val)) { + return val; + } + return {}; + }; + + config.batch_size = readPropertyOr(".batch.size", datasize_parser, "100 KB"); + config.flush_period = readPropertyOr(".flush.period", TimePeriodValue::fromString, "5 s").getMilliseconds(); + config.rate_limit = readPropertyOr(".rate.limit", TimePeriodValue::fromString, "10 min").getMilliseconds(); + config.buffer_limit = readPropertyOr(".buffer.limit", datasize_parser, "1 MB"); + config.level = readPropertyOr(".level", utils::parse_log_level, "trace"); + config.ssl_service_name = logger_properties->getString(prop_name_prefix + ".ssl.context.service"); + + return std::make_shared<AlertSink>(std::move(config), std::move(logger)); +} + +void AlertSink::initialize(core::controller::ControllerServiceProvider* controller, std::shared_ptr<AgentIdentificationProvider> agent_id) { + auto services = std::make_unique<Services>(); + + services->agent_id = std::move(agent_id); + + if (config_.ssl_service_name) { + if (!controller) { + logger_->log_error("Could not find service '%s': no service provider", config_.ssl_service_name.value()); + return; + } + if (auto service = controller->getControllerService(config_.ssl_service_name.value())) { + if (auto ssl_service = std::dynamic_pointer_cast<controllers::SSLContextService>(service)) { + services->ssl_service = ssl_service; + } else { + logger_->log_error("Service '%s' is not an SSLContextService", config_.ssl_service_name.value()); + return; + } + } else { + logger_->log_error("Could not find service '%s'", config_.ssl_service_name.value()); + return; + } + } + + services.reset(services_.exchange(services.release())); +} + +void AlertSink::sink_it_(const spdlog::details::log_msg& msg) { + // this method is protected upstream in base_sink by a mutex + + std::match_results<std::string_view::const_iterator> match; + std::string_view payload(msg.payload.data(), msg.payload.size()); + if (!std::regex_match(payload.begin(), payload.end(), match, config_.filter)) { + return; + } + size_t hash = 0; + for (size_t idx = 1; idx < match.size(); ++idx) { + std::string_view submatch; + if (match[idx].first != match[idx].second) { + // TODO(adebreceni): std::string_view(It begin, It end) is not yet supported on all platforms Review Comment: added compiler version ########## libminifi/include/core/logging/alert/AlertSink.h: ########## @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <deque> +#include <mutex> +#include <unordered_set> +#include <regex> +#include <utility> +#include <string> +#include <memory> + +#include "controllers/SSLContextService.h" +#include "core/controller/ControllerServiceProvider.h" +#include "core/logging/LoggerProperties.h" +#include "utils/ThreadPool.h" +#include "utils/StagingQueue.h" +#include "properties/Configure.h" +#include "spdlog/sinks/base_sink.h" + + +namespace org::apache::nifi::minifi::core::logging { + +class AlertSink : public spdlog::sinks::base_sink<std::mutex> { + struct Config { + std::string url; + std::optional<std::string> ssl_service_name; + int batch_size; + std::chrono::milliseconds flush_period; + std::chrono::milliseconds rate_limit; + int buffer_limit; + std::regex filter; + spdlog::level::level_enum level; + }; + + struct Services { + std::shared_ptr<controllers::SSLContextService> ssl_service; + std::shared_ptr<AgentIdentificationProvider> agent_id; + }; + + struct LogBuffer { + size_t size_{0}; + std::deque<std::pair<std::string, size_t>> data_; + + static LogBuffer allocate(size_t size); + LogBuffer commit(); + [[nodiscard]] + size_t size() const; + }; + + class LiveLogSet { + std::chrono::milliseconds lifetime_{}; + std::unordered_set<size_t> ignored_; + std::deque<std::pair<std::chrono::milliseconds, size_t>> ordered_; Review Comment: renamed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
