This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch cherry-pick-net-dashboard in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bfdb906ba54a6d7cf1674e24a3e534cfc709449f Author: Liu Xuxin <[email protected]> AuthorDate: Sun Mar 12 22:49:55 2023 +0800 cherry pick network metrics --- .../iotdb/confignode/service/ConfigNode.java | 4 + .../metrics/metricsets/net/INetMetricManager.java | 58 ++++++ .../metricsets/net/LinuxNetMetricManager.java | 197 +++++++++++++++++++++ .../metricsets/net/MacNetMetricManager.java | 22 +++ .../iotdb/metrics/metricsets/net/NetMetrics.java | 98 ++++++++++ .../metricsets/net/WindowsNetMetricManager.java | 22 +++ .../db/service/metrics/DataNodeMetricsHelper.java | 2 + 7 files changed, 403 insertions(+) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 090d1f4bb0..f69e2c13eb 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -46,6 +46,7 @@ import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics; import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics; import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics; +import org.apache.iotdb.metrics.metricsets.net.NetMetrics; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -229,6 +230,9 @@ public class ConfigNode implements ConfigNodeMBean { MetricService.getInstance().addMetricSet(new ProcessMetrics()); MetricService.getInstance().addMetricSet(new SystemMetrics(false)); MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.CN_ROLE)); + MetricService.getInstance().addMetricSet(new NetMetrics()); + + LOGGER.info("Successfully setup internal services."); } private void initConfigManager() { diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java new file mode 100644 index 0000000000..f6ae15d37c --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.metricsets.net; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +public interface INetMetricManager { + static INetMetricManager getNetMetricManager() { + String os = System.getProperty("os.name").toLowerCase(); + + if (os.startsWith("windows")) { + return new WindowsNetMetricManager(); + } else if (os.startsWith("linux")) { + return new LinuxNetMetricManager(); + } else { + return new MacNetMetricManager(); + } + } + + default Map<String, Long> getReceivedByte() { + return Collections.emptyMap(); + } + + default Map<String, Long> getTransmittedBytes() { + return Collections.emptyMap(); + } + + default Map<String, Long> getReceivedPackets() { + return Collections.emptyMap(); + } + + default Map<String, Long> getTransmittedPackets() { + return Collections.emptyMap(); + } + + default Set<String> getIfaceSet() { + return Collections.emptySet(); + } +} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java new file mode 100644 index 0000000000..4e3fab7061 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.metricsets.net; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Scanner; +import java.util.Set; +import java.util.stream.Collectors; + +public class LinuxNetMetricManager implements INetMetricManager { + private final Logger log = LoggerFactory.getLogger(LinuxNetMetricManager.class); + + @SuppressWarnings("squid:S1075") + private static final String IFACE_ID_PATH = "/sys/class/net/"; + + @SuppressWarnings("squid:S1075") + private static final String NET_STATUS_PATH = "/proc/net/dev"; + + private static final String BYTES = "bytes"; + private static final String PACKETS = "packets"; + private static final long UPDATE_INTERVAL = 10_000L; + + private static final int IFACE_NAME_INDEX = 0; + // initialized after reading status file + private int receivedBytesIndex = 0; + private int transmittedBytesIndex = 0; + private int receivedPacketsIndex = 0; + private int transmittedPacketsIndex = 0; + private Set<String> ifaceSet; + + private final Map<String, Long> receivedBytesMapForIface; + private final Map<String, Long> transmittedBytesMapForIface; + private final Map<String, Long> receivedPacketsMapForIface; + private final Map<String, Long> transmittedPacketsMapForIface; + + public LinuxNetMetricManager() { + collectIfaces(); + // leave one entry to avoid hashmap resizing + receivedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1); + transmittedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1); + receivedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1); + transmittedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1); + collectNetStatusIndex(); + } + + private long lastUpdateTime = 0L; + + @Override + public Set<String> getIfaceSet() { + checkUpdate(); + return ifaceSet; + } + + @Override + public Map<String, Long> getReceivedByte() { + checkUpdate(); + return receivedBytesMapForIface; + } + + @Override + public Map<String, Long> getTransmittedBytes() { + checkUpdate(); + return transmittedBytesMapForIface; + } + + @Override + public Map<String, Long> getReceivedPackets() { + checkUpdate(); + return receivedPacketsMapForIface; + } + + @Override + public Map<String, Long> getTransmittedPackets() { + checkUpdate(); + return transmittedPacketsMapForIface; + } + + private void checkUpdate() { + if (System.currentTimeMillis() - lastUpdateTime >= UPDATE_INTERVAL) { + updateNetStatus(); + } + } + + private void collectIfaces() { + File ifaceIdFolder = new File(IFACE_ID_PATH); + if (!ifaceIdFolder.exists()) { + ifaceSet = Collections.emptySet(); + log.warn("Cannot find {}", IFACE_ID_PATH); + return; + } + ifaceSet = + new ArrayList<>(Arrays.asList(Objects.requireNonNull(ifaceIdFolder.listFiles()))) + .stream().map(File::getName).collect(Collectors.toSet()); + } + + private void collectNetStatusIndex() { + File netStatusFile = new File(NET_STATUS_PATH); + if (!netStatusFile.exists()) { + log.warn("Cannot find {}", NET_STATUS_PATH); + return; + } + try (FileInputStream inputStream = new FileInputStream(netStatusFile)) { + Scanner scanner = new Scanner(inputStream); + // skip the first line + scanner.nextLine(); + String headerLine = scanner.nextLine(); + String[] seperatedHeaderLine = headerLine.split("\\|"); + String[] receiveStatusHeader = seperatedHeaderLine[1].split("\\s+"); + String[] transmitStatusHeader = seperatedHeaderLine[2].split("\\s+"); + for (int i = 0, length = receiveStatusHeader.length; i < length; ++i) { + if (receiveStatusHeader[i].equals(BYTES)) { + receivedBytesIndex = i + 1; + } else if (receiveStatusHeader[i].equals(PACKETS)) { + receivedPacketsIndex = i + 1; + } + } + for (int i = 0, length = transmitStatusHeader.length; i < length; ++i) { + if (transmitStatusHeader[i].equals(BYTES)) { + transmittedBytesIndex = i + length + 1; + } else if (transmitStatusHeader[i].equals(PACKETS)) { + transmittedPacketsIndex = i + length + 1; + } + } + } catch (IOException e) { + log.error("Meets exception when reading {}", NET_STATUS_PATH, e); + } + } + + private void updateNetStatus() { + lastUpdateTime = System.currentTimeMillis(); + File netStatusFile = new File(NET_STATUS_PATH); + if (!netStatusFile.exists()) { + return; + } + try (FileInputStream inputStream = new FileInputStream(netStatusFile)) { + Scanner scanner = new Scanner(inputStream); + // skip the starting two lines + // because they are the meta info + scanner.nextLine(); + scanner.nextLine(); + + // reading the actual status info for iface + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + // we wrap the string array as array list to execute the removing step + List<String> statusInfoAsList = new ArrayList<>(Arrays.asList(line.split("\\s"))); + // remove all empty string + statusInfoAsList.removeIf(x -> x.equals("")); + + String iface = statusInfoAsList.get(IFACE_NAME_INDEX); + // since the read iface format is "IFACE:" + // we need to remove the last letter + iface = iface.substring(0, iface.length() - 1); + + long receivedBytes = Long.parseLong(statusInfoAsList.get(receivedBytesIndex)); + receivedBytesMapForIface.put(iface, receivedBytes); + long transmittedBytes = Long.parseLong(statusInfoAsList.get(transmittedBytesIndex)); + transmittedBytesMapForIface.put(iface, transmittedBytes); + long receivedPackets = Long.parseLong(statusInfoAsList.get(receivedPacketsIndex)); + receivedPacketsMapForIface.put(iface, receivedPackets); + long transmittedPackets = Long.parseLong(statusInfoAsList.get(transmittedPacketsIndex)); + transmittedPacketsMapForIface.put(iface, transmittedPackets); + } + } catch (IOException e) { + log.error("Meets error when reading {} for net status", NET_STATUS_PATH, e); + } + } +} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java new file mode 100644 index 0000000000..bb3a31fba2 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.metricsets.net; + +public class MacNetMetricManager implements INetMetricManager {} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java new file mode 100644 index 0000000000..e1e565c049 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.metricsets.net; + +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Set; + +public class NetMetrics implements IMetricSet { + private final INetMetricManager netMetricManager = INetMetricManager.getNetMetricManager(); + + private static final String RECEIVED_BYTES = "received_bytes"; + private static final String RECEIVED_PACKETS = "received_packets"; + private static final String TRANSMITTED_BYTES = "transmitted_bytes"; + private static final String TRANSMITTED_PACKETS = "transmitted_packets"; + + private static final String TYPE = "type"; + private static final String IFACE_NAME = "iface_name"; + private static final String RECEIVE = "receive"; + private static final String TRANSMIT = "transmit"; + + @Override + public void bindTo(AbstractMetricService metricService) { + // metrics for net + Set<String> ifaceSet = netMetricManager.getIfaceSet(); + for (String iface : ifaceSet) { + metricService.createAutoGauge( + RECEIVED_BYTES, + MetricLevel.IMPORTANT, + netMetricManager, + x -> x.getReceivedByte().get(iface), + TYPE, + RECEIVE, + IFACE_NAME, + iface); + metricService.createAutoGauge( + TRANSMITTED_BYTES, + MetricLevel.IMPORTANT, + netMetricManager, + x -> x.getTransmittedBytes().get(iface), + TYPE, + TRANSMIT, + IFACE_NAME, + iface); + metricService.createAutoGauge( + RECEIVED_PACKETS, + MetricLevel.IMPORTANT, + netMetricManager, + x -> x.getReceivedPackets().get(iface), + TYPE, + RECEIVE, + IFACE_NAME, + iface); + metricService.createAutoGauge( + TRANSMITTED_PACKETS, + MetricLevel.IMPORTANT, + netMetricManager, + x -> x.getTransmittedPackets().get(iface), + TYPE, + TRANSMIT, + IFACE_NAME, + iface); + } + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + Set<String> ifaceSet = netMetricManager.getIfaceSet(); + for (String iface : ifaceSet) { + metricService.remove(MetricType.AUTO_GAUGE, RECEIVED_BYTES, TYPE, RECEIVE, IFACE_NAME, iface); + metricService.remove(MetricType.AUTO_GAUGE, TRANSMIT, TYPE, TRANSMIT, IFACE_NAME, iface); + metricService.remove( + MetricType.AUTO_GAUGE, RECEIVED_PACKETS, TYPE, RECEIVE, IFACE_NAME, iface); + metricService.remove( + MetricType.AUTO_GAUGE, TRANSMITTED_PACKETS, TYPE, TRANSMIT, IFACE_NAME, iface); + } + } +} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java new file mode 100644 index 0000000000..c3ecb4b8d5 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.metricsets.net; + +public class WindowsNetMetricManager implements INetMetricManager {} diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index 664b0c3a67..110d4cfa4c 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet; import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics; import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics; import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics; +import org.apache.iotdb.metrics.metricsets.net.NetMetrics; public class DataNodeMetricsHelper { /** Bind predefined metric sets into DataNode. */ @@ -43,6 +44,7 @@ public class DataNodeMetricsHelper { MetricService.getInstance().addMetricSet(new ProcessMetrics()); MetricService.getInstance().addMetricSet(new SystemMetrics(true)); MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE)); + MetricService.getInstance().addMetricSet(new NetMetrics()); // bind query related metrics MetricService.getInstance().addMetricSet(new QueryPlanCostMetricSet());
