This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 93f1b63712a9289a24687402dda22f74732e304d Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Sep 12 15:59:01 2023 +0200 MINIFICPP-2182 Add System Load Avg field field to C2 and Prometheus metrics Closes #1636 Signed-off-by: Marton Szasz <[email protected]> --- C2.md | 3 ++- METRICS.md | 11 ++++++----- .../cluster/checkers/PrometheusChecker.py | 2 +- .../C2VerifyResourceConsumptionInHeartbeat.cpp | 5 +++++ .../include/core/state/nodes/DeviceInformation.h | 2 ++ libminifi/include/utils/OsUtils.h | 1 + .../src/core/state/nodes/DeviceInformation.cpp | 23 ++++++++++++++++++++-- libminifi/src/utils/OsUtils.cpp | 14 +++++++++++++ 8 files changed, 52 insertions(+), 9 deletions(-) diff --git a/C2.md b/C2.md index 335f59b87..c0690dd0c 100644 --- a/C2.md +++ b/C2.md @@ -157,7 +157,8 @@ configuration produces the following JSON: "RuntimeMetrics": { "deviceInfo": { "systemInfo": { - "cpuUtilization": -1, + "cpuLoadAverage": 2.59521484375, + "cpuUtilization": 0.1188118811881188, "machineArch": "x86_64", "memoryUsage": 13103550464, "operatingSystem": "Linux", diff --git a/METRICS.md b/METRICS.md index d6d63481a..7aa53fda7 100644 --- a/METRICS.md +++ b/METRICS.md @@ -170,11 +170,12 @@ RepositoryMetrics is a system level metric that reports metrics for the register DeviceInfoNode is a system level metric that reports metrics about the system resources used and available -| Metric name | Labels | Description | -|-----------------|--------------|-------------------------------------| -| physical_mem | - | Physical memory available | -| memory_usage | - | Physical memory usage of the system | -| cpu_utilization | - | CPU utilized by the system | +| Metric name | Labels | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------| +| physical_mem | - | Physical memory available | +| memory_usage | - | Physical memory usage of the system | +| cpu_utilization | - | CPU utilized by the system | +| cpu_load_average | - | The number of processes in the system run queue averaged over the last minute. This metrics is not available on Windows. | ### FlowInformation diff --git a/docker/test/integration/cluster/checkers/PrometheusChecker.py b/docker/test/integration/cluster/checkers/PrometheusChecker.py index 2c1f1140c..050310717 100644 --- a/docker/test/integration/cluster/checkers/PrometheusChecker.py +++ b/docker/test/integration/cluster/checkers/PrometheusChecker.py @@ -72,7 +72,7 @@ class PrometheusChecker: self.verify_metric_exists('minifi_is_running', 'FlowInformation', {'component_name': 'FlowController'}) def verify_device_info_node_metrics(self): - return self.verify_metrics_exist(['minifi_physical_mem', 'minifi_memory_usage', 'minifi_cpu_utilization'], 'DeviceInfoNode') + return self.verify_metrics_exist(['minifi_physical_mem', 'minifi_memory_usage', 'minifi_cpu_utilization', 'minifi_cpu_load_average'], 'DeviceInfoNode') def verify_agent_status_metrics(self): label_list = [{'repository_name': 'flowfile'}, {'repository_name': 'content'}] diff --git a/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp b/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp index 35eaaad0c..8302d6175 100644 --- a/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp +++ b/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp @@ -71,6 +71,11 @@ class ResourceConsumptionInHeartbeatHandler : public HeartbeatHandler { assert(system_info.HasMember("machineArch")); assert(system_info["machineArch"].GetStringLength() > 0); + +#ifndef WIN32 + assert(system_info.HasMember("cpuLoadAverage")); + assert(system_info["cpuLoadAverage"].GetDouble() >= 0.0); +#endif } static void verifyProcessResourceConsumption(const rapidjson::Document& root, bool firstCall) { diff --git a/libminifi/include/core/state/nodes/DeviceInformation.h b/libminifi/include/core/state/nodes/DeviceInformation.h index a44a6887b..bb55818d7 100644 --- a/libminifi/include/core/state/nodes/DeviceInformation.h +++ b/libminifi/include/core/state/nodes/DeviceInformation.h @@ -44,6 +44,7 @@ #include <string> #include <vector> #include <utility> +#include <optional> #include "core/state/nodes/MetricsBase.h" #include "Connection.h" @@ -109,6 +110,7 @@ class DeviceInfoNode : public DeviceInformation { static SerializedResponseNode serializeTotalPhysicalMemoryInformation(); static SerializedResponseNode serializePhysicalMemoryUsageInformation(); static SerializedResponseNode serializeSystemCPUUsageInformation(); + static std::optional<SerializedResponseNode> serializeCPULoadAverageInformation(); static SerializedResponseNode serializeArchitectureInformation(); static SerializedResponseNode serializeSystemInfo(); SerializedResponseNode serializeHostNameInfo() const; diff --git a/libminifi/include/utils/OsUtils.h b/libminifi/include/utils/OsUtils.h index a71172333..837806749 100644 --- a/libminifi/include/utils/OsUtils.h +++ b/libminifi/include/utils/OsUtils.h @@ -54,5 +54,6 @@ extern std::string resolve_common_identifiers(const std::string &id); #endif std::optional<std::string> getHostName(); +std::optional<double> getSystemLoadAverage(); } // namespace org::apache::nifi::minifi::utils::OsUtils diff --git a/libminifi/src/core/state/nodes/DeviceInformation.cpp b/libminifi/src/core/state/nodes/DeviceInformation.cpp index 65e02370d..1496c5826 100644 --- a/libminifi/src/core/state/nodes/DeviceInformation.cpp +++ b/libminifi/src/core/state/nodes/DeviceInformation.cpp @@ -240,11 +240,17 @@ std::vector<PublishedMetric> DeviceInfoNode::calculateMetrics() { SerializedResponseNode cpu_usage; cpu_usage.name = "cpuUtilization"; cpu_usage.value = system_cpu_usage; - return { + std::vector<PublishedMetric> metrics = { {"physical_mem", static_cast<double>(utils::OsUtils::getSystemTotalPhysicalMemory()), {{"metric_class", "DeviceInfoNode"}}}, {"memory_usage", static_cast<double>(utils::OsUtils::getSystemPhysicalMemoryUsage()), {{"metric_class", "DeviceInfoNode"}}}, {"cpu_utilization", system_cpu_usage, {{"metric_class", "DeviceInfoNode"}}}, }; + + if (auto system_load_average = utils::OsUtils::getSystemLoadAverage()) { + metrics.push_back({"cpu_load_average", *system_load_average, {{"metric_class", "DeviceInfoNode"}}}); + } + + return metrics; } SerializedResponseNode DeviceInfoNode::serializeIdentifier() const { @@ -283,8 +289,16 @@ SerializedResponseNode DeviceInfoNode::serializeArchitectureInformation() { return {.name = "machineArch", .value = utils::OsUtils::getMachineArchitecture()}; } +std::optional<SerializedResponseNode> DeviceInfoNode::serializeCPULoadAverageInformation() { + if (auto system_load_average = utils::OsUtils::getSystemLoadAverage()) { + return SerializedResponseNode{.name = "cpuLoadAverage", .value = *system_load_average}; + } + + return std::nullopt; +} + SerializedResponseNode DeviceInfoNode::serializeSystemInfo() { - return { + SerializedResponseNode system_info = { .name = "systemInfo", .children = { serializeVCoreInfo(), @@ -295,6 +309,11 @@ SerializedResponseNode DeviceInfoNode::serializeSystemInfo() { serializeSystemCPUUsageInformation() } }; + + if (auto cpu_load_average = serializeCPULoadAverageInformation()) { + system_info.children.push_back(*cpu_load_average); + } + return system_info; } SerializedResponseNode DeviceInfoNode::serializeHostNameInfo() const { diff --git a/libminifi/src/utils/OsUtils.cpp b/libminifi/src/utils/OsUtils.cpp index 4c2fc9a00..e1c03a947 100644 --- a/libminifi/src/utils/OsUtils.cpp +++ b/libminifi/src/utils/OsUtils.cpp @@ -26,6 +26,7 @@ #ifdef __linux__ #include <sys/sysinfo.h> +#include <cstdlib> #include <optional> #include <sstream> #endif @@ -336,4 +337,17 @@ std::optional<std::string> OsUtils::getHostName() { return {hostname}; } +std::optional<double> OsUtils::getSystemLoadAverage() { +#ifndef WIN32 + double load_avg[1]; + auto numSamples = getloadavg(load_avg, 1); + if (numSamples == -1) { + return std::nullopt; + } + return load_avg[0]; +#else + return std::nullopt; +#endif +} + } // namespace org::apache::nifi::minifi::utils
