This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch ci/add-metric-it in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae3ff5fb5c0709d8247adc9dab6b1ec34fe2a6d8 Author: spricoder <[email protected]> AuthorDate: Wed Nov 1 00:59:40 2023 +0800 Add IoTDBMetricIT --- .../org/apache/iotdb/it/env/cluster/EnvUtils.java | 2 +- .../it/env/cluster/config/MppConfigNodeConfig.java | 8 +++ .../it/env/cluster/config/MppDataNodeConfig.java | 8 +++ .../iotdb/it/env/cluster/env/AbstractEnv.java | 33 +++++------- .../it/env/cluster/node/AbstractNodeWrapper.java | 7 +++ .../it/env/cluster/node/ConfigNodeWrapper.java | 2 + .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 2 + .../env/remote/config/RemoteConfigNodeConfig.java | 9 +++- .../it/env/remote/config/RemoteDataNodeConfig.java | 9 +++- .../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +++ .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 32 +++++++++++ .../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 + .../apache/iotdb/itbase/env/ConfigNodeConfig.java | 6 ++- .../apache/iotdb/itbase/env/DataNodeConfig.java | 6 ++- .../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 63 ++++++++++++++++++++++ .../apache/iotdb/commons/conf/IoTDBConstant.java | 5 ++ pom.xml | 1 + 17 files changed, 175 insertions(+), 26 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 8036b0faa4a..66af6e748cd 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -72,7 +72,7 @@ public class EnvUtils { continue; } List<Integer> requiredPorts = - IntStream.rangeClosed(randomPortStart, randomPortStart + 9) + IntStream.rangeClosed(randomPortStart, randomPortStart + 10) .boxed() .collect(Collectors.toList()); if (checkPortsAvailable(requiredPorts)) { diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java index e8ee382a799..b81ed4cbbb6 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppConfigNodeConfig.java @@ -22,6 +22,7 @@ package org.apache.iotdb.it.env.cluster.config; import org.apache.iotdb.itbase.env.ConfigNodeConfig; import java.io.IOException; +import java.util.List; public class MppConfigNodeConfig extends MppBaseConfig implements ConfigNodeConfig { @@ -48,4 +49,11 @@ public class MppConfigNodeConfig extends MppBaseConfig implements ConfigNodeConf + persistentConfig.getClass().getCanonicalName()); } } + + @Override + public ConfigNodeConfig setMetricReporterType(List<String> metricReporterTypes) { + properties.setProperty( + "cn_metric_reporter_list", "[" + String.join(",", metricReporterTypes) + "]"); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index ea65fa2b9cd..1c7d11377af 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -22,6 +22,7 @@ package org.apache.iotdb.it.env.cluster.config; import org.apache.iotdb.itbase.env.DataNodeConfig; import java.io.IOException; +import java.util.List; public class MppDataNodeConfig extends MppBaseConfig implements DataNodeConfig { @@ -48,4 +49,11 @@ public class MppDataNodeConfig extends MppBaseConfig implements DataNodeConfig { + persistentConfig.getClass().getCanonicalName()); } } + + @Override + public DataNodeConfig setMetricReporterType(List<String> metricReporterTypes) { + properties.setProperty( + "dn_metric_reporter_list", "[" + String.join(",", metricReporterTypes) + "]"); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 6edfe0122ff..4b0d3798839 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -34,11 +34,7 @@ import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.pool.ISessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.EnvUtils; -import org.apache.iotdb.it.env.cluster.config.MppClusterConfig; -import org.apache.iotdb.it.env.cluster.config.MppCommonConfig; -import org.apache.iotdb.it.env.cluster.config.MppConfigNodeConfig; -import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig; -import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; +import org.apache.iotdb.it.env.cluster.config.*; import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -46,11 +42,7 @@ import org.apache.iotdb.it.framework.IoTDBTestLogger; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.itbase.env.BaseNodeWrapper; import org.apache.iotdb.itbase.env.ClusterConfig; -import org.apache.iotdb.itbase.runtime.ClusterTestConnection; -import org.apache.iotdb.itbase.runtime.NodeConnection; -import org.apache.iotdb.itbase.runtime.ParallelRequestDelegate; -import org.apache.iotdb.itbase.runtime.RequestDelegate; -import org.apache.iotdb.itbase.runtime.SerialRequestDelegate; +import org.apache.iotdb.itbase.runtime.*; import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.Constant; import org.apache.iotdb.jdbc.IoTDBConnection; @@ -67,22 +59,13 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.NODE_NETWORK_TIMEOUT_MS; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.NODE_START_TIMEOUT; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.PROBE_TIMEOUT_MS; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.TEMPLATE_NODE_LIB_PATH; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.TEMPLATE_NODE_PATH; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.*; import static org.apache.iotdb.jdbc.Config.VERSION; public abstract class AbstractEnv implements BaseEnv { @@ -121,6 +104,14 @@ public abstract class AbstractEnv implements BaseEnv { return clusterConfig; } + @Override + public String getMetricPrometheusReporterContent() { + DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return getUrlContent( + Config.IOTDB_URL_PREFIX + dataNode.getIp() + ":" + dataNode.getMetricPort() + "/metrics"); + } + protected void initEnvironment(int configNodesNum, int dataNodesNum) { initEnvironment(configNodesNum, dataNodesNum, 30); } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 187a819f332..433a180fa4f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -123,6 +123,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { private Process instance; private final String nodeAddress; private int nodePort; + private int metricPort; private long startTime; /** @@ -161,6 +162,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { this.portList = portList; this.nodeAddress = "127.0.0.1"; this.nodePort = portList[0]; + this.metricPort = portList[portList.length - 2]; jmxPort = this.portList[portList.length - 1]; // these properties can't be mutated. immutableCommonProperties.setProperty(UDF_LIB_DIR, MppBaseConfig.NULL_VALUE); @@ -470,6 +472,11 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { return this.nodePort; } + @Override + public final int getMetricPort() { + return this.metricPort; + } + public void setPort(int port) { this.nodePort = port; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index 8aec051dc9b..96b93e754b6 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -168,6 +168,8 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper { mutableNodeProperties.setProperty(IoTDBConstant.CN_INTERNAL_PORT, String.valueOf(getPort())); mutableNodeProperties.setProperty( IoTDBConstant.CN_CONSENSUS_PORT, String.valueOf(this.consensusPort)); + mutableNodeProperties.setProperty( + IoTDBConstant.CN_METRIC_PROMETHEUS_REPORTER_PORT, String.valueOf(super.getMetricPort())); } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index e7543813599..5f6d45446de 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -195,6 +195,8 @@ public class DataNodeWrapper extends AbstractNodeWrapper { mutableNodeProperties.setProperty(IoTDBConstant.DN_RPC_ADDRESS, super.getIp()); mutableNodeProperties.setProperty(IoTDBConstant.DN_RPC_PORT, String.valueOf(super.getPort())); mutableNodeProperties.setProperty(IoTDBConstant.DN_INTERNAL_ADDRESS, this.internalAddress); + mutableNodeProperties.setProperty( + IoTDBConstant.DN_METRIC_PROMETHEUS_REPORTER_PORT, String.valueOf(super.getMetricPort())); mutableNodeProperties.setProperty( IoTDBConstant.DN_INTERNAL_PORT, String.valueOf(this.internalPort)); mutableNodeProperties.setProperty( diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java index c10eaa9afb6..5c40fd6ef6b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java @@ -20,4 +20,11 @@ package org.apache.iotdb.it.env.remote.config; import org.apache.iotdb.itbase.env.ConfigNodeConfig; -public class RemoteConfigNodeConfig implements ConfigNodeConfig {} +import java.util.List; + +public class RemoteConfigNodeConfig implements ConfigNodeConfig { + @Override + public ConfigNodeConfig setMetricReporterType(List<String> metricReporterTypes) { + return this; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index dbd640fdff4..6bb6cfbcef4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -20,4 +20,11 @@ package org.apache.iotdb.it.env.remote.config; import org.apache.iotdb.itbase.env.DataNodeConfig; -public class RemoteDataNodeConfig implements DataNodeConfig {} +import java.util.List; + +public class RemoteDataNodeConfig implements DataNodeConfig { + @Override + public DataNodeConfig setMetricReporterType(List<String> metricReporterTypes) { + return this; + } +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java index da0a9e0dcef..3a100611aa2 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java @@ -55,6 +55,7 @@ public class RemoteServerEnv implements BaseEnv { private final String ip_addr = System.getProperty("RemoteIp", "127.0.0.1"); private final String port = System.getProperty("RemotePort", "6667"); + private final String metricPort = System.getProperty("RemoteMetricPort", "9091"); private final String user = System.getProperty("RemoteUser", "root"); private final String password = System.getProperty("RemotePassword", "root"); private IClientManager<TEndPoint, SyncConfigNodeIServiceClient> clientManager; @@ -97,6 +98,11 @@ public class RemoteServerEnv implements BaseEnv { return clusterConfig; } + @Override + public String getMetricPrometheusReporterContent() { + return getUrlContent(Config.IOTDB_URL_PREFIX + ip_addr + ":" + metricPort + "/metrics"); + } + @Override public Connection getConnection(String username, String password) throws SQLException { Connection connection = null; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java index 5e3f18f450f..da57021f2bd 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java @@ -29,7 +29,12 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.jdbc.Constant; import org.apache.iotdb.rpc.IoTDBConnectionException; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; import java.sql.Connection; import java.sql.SQLException; import java.util.List; @@ -62,6 +67,33 @@ public interface BaseEnv { /** Return the {@link ClusterConfig} for developers to set values before test. */ ClusterConfig getConfig(); + default String getUrlContent(String urlStr) { + StringBuilder sb = new StringBuilder(); + try { + URL url = new URL(urlStr); + HttpURLConnection httpConnection = (HttpURLConnection) url.openConnection(); + if (httpConnection.getResponseCode() == HttpURLConnection.HTTP_OK) { + InputStream in = httpConnection.getInputStream(); + InputStreamReader isr = new InputStreamReader(in); + BufferedReader bufr = new BufferedReader(isr); + String str; + while ((str = bufr.readLine()) != null) { + sb.append(str); + } + bufr.close(); + } else { + return null; + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return sb.toString(); + } + + /** Return the content of prometheus. */ + String getMetricPrometheusReporterContent(); + default Connection getConnection() throws SQLException { return getConnection("root", "root"); } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java index 0f560af8e6f..165ed3fd62f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java @@ -35,6 +35,8 @@ public interface BaseNodeWrapper { int getPort(); + int getMetricPort(); + String getId(); String getIpAndPortString(); diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java index 658c59bf62a..bf7179ef702 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/ConfigNodeConfig.java @@ -19,5 +19,9 @@ package org.apache.iotdb.itbase.env; +import java.util.List; + /** This interface is used to handle properties in iotdb-confignode.properties. */ -public interface ConfigNodeConfig {} +public interface ConfigNodeConfig { + ConfigNodeConfig setMetricReporterType(List<String> metricReporterTypes); +} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index 80bf0ae5aa3..2778160d4d9 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -19,5 +19,9 @@ package org.apache.iotdb.itbase.env; +import java.util.List; + /** This interface is used to handle properties in iotdb-datanode.properties. */ -public interface DataNodeConfig {} +public interface DataNodeConfig { + DataNodeConfig setMetricReporterType(List<String> metricReporterTypes); +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java new file mode 100644 index 00000000000..2211929beb3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.metric; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBMetricIT { + @BeforeClass + public static void setUp() throws Exception { + // Start ConfigNode with Prometheus reporter up + EnvFactory.getEnv() + .getConfig() + .getConfigNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + // Start DataNode with Prometheus reporter up + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testPrometheusReporter() { + Assert.assertNotNull(EnvFactory.getEnv().getMetricPrometheusReporterContent()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 87a513049da..479c070b9a3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -62,6 +62,11 @@ public class IoTDBConstant { public static final String CN_INTERNAL_ADDRESS = "cn_internal_address"; public static final String DN_INTERNAL_ADDRESS = "dn_internal_address"; + public static final String CN_METRIC_PROMETHEUS_REPORTER_PORT = + "cn_metric_prometheus_reporter_port"; + public static final String DN_METRIC_PROMETHEUS_REPORTER_PORT = + "dn_metric_prometheus_reporter_port"; + public static final String CN_INTERNAL_PORT = "cn_internal_port"; public static final String DN_INTERNAL_PORT = "dn_internal_port"; public static final String CN_CONSENSUS_PORT = "cn_consensus_port"; diff --git a/pom.xml b/pom.xml index e1a6c0163f5..5c773f31ae1 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ <name>Apache IoTDB Project Parent POM</name> <description>This is the top level project that builds, packages the tsfile, iotdb engine, jdbc, and integration libs.</description> <modules> + <module>integration-test</module> <module>iotdb-api</module> <module>iotdb-client</module> <module>iotdb-connector</module>
