This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch addCIForAINode
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/addCIForAINode by this push:
new 43abf9864c8 fix the IT procedure of AINode
43abf9864c8 is described below
commit 43abf9864c8f078523a2718a8b715451b7e533d8
Author: ycycse <[email protected]>
AuthorDate: Mon Sep 2 22:11:32 2024 +0800
fix the IT procedure of AINode
---
integration-test/src/assembly/mpp-test.xml | 7 +-
.../iotdb/it/env/cluster/ClusterConstant.java | 1 +
.../org/apache/iotdb/it/env/cluster/env/AIEnv.java | 2 +-
.../iotdb/it/env/cluster/node/AINodeWrapper.java | 75 +++++++++-------------
.../it/env/cluster/node/AbstractNodeWrapper.java | 4 +-
.../iotdb/{mlnode => ainode}/it/AIEnvIT.java | 0
iotdb-core/ainode/pyproject.toml | 2 +-
.../ainode/resources/conf/iotdb-ainode.properties | 8 +--
8 files changed, 46 insertions(+), 53 deletions(-)
diff --git a/integration-test/src/assembly/mpp-test.xml
b/integration-test/src/assembly/mpp-test.xml
index 656429c6153..9a4a8727ea2 100644
--- a/integration-test/src/assembly/mpp-test.xml
+++ b/integration-test/src/assembly/mpp-test.xml
@@ -58,7 +58,12 @@
</fileSet>
<fileSet>
<outputDirectory>sbin</outputDirectory>
-
<directory>${{project.basedir}/../iotdb-core/ainode/resources/sbin</directory>
+
<directory>${project.basedir}/../iotdb-core/ainode/resources/sbin</directory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>venv</outputDirectory>
+ <directory>${project.basedir}/../iotdb-core/ainode/venv</directory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
index cc231260d0f..b3802f3ea8d 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
@@ -182,6 +182,7 @@ public class ClusterConstant {
// Paths
public static final String USER_DIR = "user.dir";
public static final String TARGET = "target";
+ public static final String PYTHON_PATH = "venv/bin/python3";
public static final String DATA_NODE_NAME = "DataNode";
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
index b01d9b5735d..48603729300 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java
@@ -8,7 +8,7 @@ public class AIEnv extends AbstractEnv {
@Override
public void initClusterEnvironment(int configNodesNum, int dataNodesNum) {
- super.initEnvironment(configNodesNum, dataNodesNum, 100, true);
+ super.initEnvironment(configNodesNum, dataNodesNum, 10000, true);
}
@Override
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
index 233347c2b06..c503dfc00e7 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java
@@ -24,21 +24,23 @@ import org.apache.iotdb.it.framework.IoTDBTestLogger;
import org.slf4j.Logger;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
-import java.nio.file.*;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.*;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.PYTHON_PATH;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR;
import static org.apache.iotdb.it.env.cluster.EnvUtils.getTimeForLogDirectory;
public class AINodeWrapper extends AbstractNodeWrapper {
private static final Logger logger = IoTDBTestLogger.logger;
- private final String testClassName;
- private final String testMethodName;
private final long startTime;
private final String seedConfigNode;
@@ -49,21 +51,18 @@ public class AINodeWrapper extends AbstractNodeWrapper {
private static final String PROPERTIES_FILE = "iotdb-ainode.properties";
public static final String CONFIG_PATH = "conf";
public static final String SCRIPT_PATH = "sbin";
- private static final String INTERPRETER_PATH = "/root/.venv/bin/python3";
- private String[] getReplaceCmd(String key, String value, String filePath) {
- String s = String.format("sed -i s/^%s=.*/%s=%s/ %s", key, key, value,
filePath);
- return s.split("\\s+");
- }
-
- private void replaceAttribute(String key, String value, String filePath,
File stdoutFile)
- throws IOException, InterruptedException {
- ProcessBuilder prepareProcessBuilder =
- new ProcessBuilder(getReplaceCmd(key, value, filePath))
- .redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutFile))
- .redirectError(ProcessBuilder.Redirect.appendTo(stdoutFile));
- Process prepareProcess = prepareProcessBuilder.start();
- prepareProcess.waitFor();
+ private void replaceAttribute(String[] keys, String[] values, String
filePath) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath,
true))) {
+ for (int i = 0; i < keys.length; i++) {
+ String line = keys[i] + "=" + values[i];
+ writer.newLine();
+ writer.write(line);
+ }
+ } catch (IOException e) {
+ logger.error(
+ "Failed to set attribute for AINode in file: {} because {}",
filePath, e.getMessage());
+ }
}
public AINodeWrapper(
@@ -75,15 +74,10 @@ public class AINodeWrapper extends AbstractNodeWrapper {
long startTime) {
super(testClassName, testMethodName, port, clusterIndex, false, startTime);
this.seedConfigNode = seedConfigNode;
- this.testClassName = testClassName;
- this.testMethodName = testMethodName;
this.startTime = startTime;
}
- private String getLogPath() {
- return getLogDirPath() + File.separator + getId() + ".log";
- }
-
+ @Override
public String getLogDirPath() {
return System.getProperty(USER_DIR)
+ File.separator
@@ -96,13 +90,6 @@ public class AINodeWrapper extends AbstractNodeWrapper {
+ getTimeForLogDirectory(startTime);
}
- private String getTestLogDirName() {
- if (testMethodName == null) {
- return testClassName;
- }
- return testClassName + "_" + testMethodName;
- }
-
@Override
public void start() {
try {
@@ -119,25 +106,25 @@ public class AINodeWrapper extends AbstractNodeWrapper {
// set attribute
replaceAttribute(
- "ain_target_config_node_list", this.seedConfigNode, propertiesFile,
stdoutFile);
- replaceAttribute(
- "ain_inference_rpc_port", Integer.toString(getPort()),
propertiesFile, stdoutFile);
+ new String[] {"ain_seed_config_node", "ain_inference_rpc_port"},
+ new String[] {this.seedConfigNode, Integer.toString(getPort())},
+ propertiesFile);
// start AINode
- List<String> start_command = new ArrayList<>();
- start_command.add(SHELL_COMMAND);
- start_command.add(filePrefix + File.separator + SCRIPT_PATH +
File.separator + SCRIPT_FILE);
- start_command.add("-i");
- start_command.add(INTERPRETER_PATH);
- start_command.add("-r");
- start_command.add("-n");
+ List<String> startCommand = new ArrayList<>();
+ startCommand.add(SHELL_COMMAND);
+ startCommand.add(filePrefix + File.separator + SCRIPT_PATH +
File.separator + SCRIPT_FILE);
+ startCommand.add("-i");
+ startCommand.add(filePrefix + File.separator + PYTHON_PATH);
+ startCommand.add("-r");
+
ProcessBuilder processBuilder =
- new ProcessBuilder(start_command)
+ new ProcessBuilder(startCommand)
.redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutFile))
.redirectError(ProcessBuilder.Redirect.appendTo(stdoutFile));
this.instance = processBuilder.start();
logger.info("In test {} {} started.", getTestLogDirName(), getId());
- } catch (IOException | InterruptedException e) {
+ } catch (Exception e) {
throw new AssertionError("Start AI Node failed. " + e + Paths.get(""));
}
}
@@ -186,7 +173,7 @@ public class AINodeWrapper extends AbstractNodeWrapper {
;
public MppJVMConfig initVMConfig() {
- throw new UnsupportedOperationException("AINode doesn't have JVM Config");
+ return null;
}
;
}
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index e6a26bd08db..bf28c407f72 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -540,7 +540,7 @@ public abstract class AbstractNodeWrapper implements
BaseNodeWrapper {
return getNodePath() + File.separator + dirName + File.separator +
fileName;
}
- private String getLogPath() {
+ protected String getLogPath() {
return getLogDirPath() + File.separator + getId() + ".log";
}
@@ -645,7 +645,7 @@ public abstract class AbstractNodeWrapper implements
BaseNodeWrapper {
output.print(sb);
}
- private String getTestLogDirName() {
+ protected String getTestLogDirName() {
if (testMethodName == null) {
return testClassName;
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java
b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AIEnvIT.java
similarity index 100%
rename from
integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java
rename to integration-test/src/test/java/org/apache/iotdb/ainode/it/AIEnvIT.java
diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml
index 0d1103ea716..6211d230136 100644
--- a/iotdb-core/ainode/pyproject.toml
+++ b/iotdb-core/ainode/pyproject.toml
@@ -50,7 +50,7 @@ python = ">=3.8, <3.13"
numpy = "^1.21.4"
pandas = "^1.3.5"
-torch = "2.1.0"
+torch = "2.2.0"
pylru = "^1.2.1"
thrift = "^0.13.0"
diff --git a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
index 52b3dbb1bb0..2b208e72125 100644
--- a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
+++ b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
@@ -19,22 +19,22 @@
# Used for indicate cluster name and distinguish different cluster.
# Datatype: string
-cluster_name=defaultCluster
+# cluster_name=defaultCluster
# ConfigNode address registered at AINode startup
# Allow modifications only before starting the service for the first time
# Datatype: String
-ain_seed_config_node=127.0.0.1:10710
+# ain_seed_config_node=127.0.0.1:10710
# Used for connection of DataNode/ConfigNode clients
# Could set 127.0.0.1(for local test) or ipv4 address
# Datatype: String
-ain_inference_rpc_address=127.0.0.1
+# ain_inference_rpc_address=127.0.0.1
# Used for connection of DataNode/ConfigNode clients
# Bind with MN_RPC_ADDRESS
# Datatype: String
-ain_inference_rpc_port=10810
+# ain_inference_rpc_port=10810
# The AINode metadata storage path.
# The starting directory of the relative path is related to the operating
system.
