This is an automated email from the ASF dual-hosted git repository. ycycse pushed a commit to branch addCIForAINode in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a42df558594aaf3a180c963caaa44690146a75d1 Author: ycycse <[email protected]> AuthorDate: Mon Sep 2 18:59:33 2024 +0800 introduce AINode to IoTDB (fix) --- .../java/org/apache/iotdb/it/env/EnvFactory.java | 2 +- .../it/env/cluster/env/{MLEnv.java => AIEnv.java} | 0 .../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +- .../{MLNodeWrapper.java => AINodeWrapper.java} | 166 ++++++--------------- .../it/env/cluster/node/AbstractNodeWrapper.java | 8 +- .../{MLClusterIT.java => AIClusterIT.java} | 0 .../iotdb/mlnode/it/{MLEnvIT.java => AIEnvIT.java} | 0 7 files changed, 57 insertions(+), 132 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java index 88ac6f46f6e..ce8046708a8 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/EnvFactory.java @@ -19,8 +19,8 @@ package org.apache.iotdb.it.env; -import org.apache.iotdb.it.env.cluster.env.Cluster1Env; import org.apache.iotdb.it.env.cluster.env.AIEnv; +import org.apache.iotdb.it.env.cluster.env.Cluster1Env; import org.apache.iotdb.it.env.cluster.env.SimpleEnv; import org.apache.iotdb.it.env.remote.env.RemoteServerEnv; import org.apache.iotdb.it.framework.IoTDBTestLogger; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MLEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java similarity index 100% rename from integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/MLEnv.java rename to integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AIEnv.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index cb7a50c44c9..482e869cf1c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -41,10 +41,10 @@ import org.apache.iotdb.isession.pool.ISessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.*; +import org.apache.iotdb.it.env.cluster.node.AINodeWrapper; import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.env.cluster.node.AINodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestLogger; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.itbase.env.BaseNodeWrapper; @@ -82,7 +82,7 @@ public abstract class AbstractEnv implements BaseEnv { private final Random rand = new Random(); protected List<ConfigNodeWrapper> configNodeWrapperList = Collections.emptyList(); protected List<DataNodeWrapper> dataNodeWrapperList = Collections.emptyList(); - protected List<MLNodeWrapper> aiNodeWrapperList = Collections.emptyList(); + protected List<AINodeWrapper> aiNodeWrapperList = Collections.emptyList(); protected String testMethodName = null; protected int index = 0; protected long startTime; @@ -151,8 +151,8 @@ public abstract class AbstractEnv implements BaseEnv { } protected void initEnvironment( - int configNodesNum, int dataNodesNum, int testWorkingRetryCount, boolean addAINode) { - this.testWorkingRetryCount = testWorkingRetryCount; + int configNodesNum, int dataNodesNum, int retryCount, boolean addAINode) { + this.retryCount = retryCount; this.configNodeWrapperList = new ArrayList<>(); this.dataNodeWrapperList = new ArrayList<>(); @@ -278,6 +278,7 @@ public abstract class AbstractEnv implements BaseEnv { seedConfigNode, testClassName, testMethodName, + index, EnvUtils.searchAvailablePorts(), startTime); aiNodeWrapperList.add(aiNodeWrapper); @@ -400,7 +401,9 @@ public abstract class AbstractEnv implements BaseEnv { @Override public void cleanClusterEnvironment() { List<AbstractNodeWrapper> allNodeWrappers = - Stream.concat(this.dataNodeWrapperList.stream(), this.configNodeWrapperList.stream(), this.aiNodeWrapperList.strea()) + Stream.concat( + dataNodeWrapperList.stream(), + Stream.concat(configNodeWrapperList.stream(), aiNodeWrapperList.stream())) .collect(Collectors.toList()); allNodeWrappers.stream() .findAny() diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java similarity index 54% rename from integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java rename to integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java index 5d713cd0dbd..233347c2b06 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/MLNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java @@ -19,11 +19,9 @@ package org.apache.iotdb.it.env.cluster.node; +import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; import org.apache.iotdb.it.framework.IoTDBTestLogger; -import org.apache.iotdb.itbase.env.BaseNodeWrapper; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.file.PathUtils; import org.slf4j.Logger; import java.io.File; @@ -31,21 +29,16 @@ import java.io.IOException; import java.nio.file.*; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import static org.apache.iotdb.it.env.cluster.ClusterConstant.*; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.ML_NODE_NAME; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME; import static org.apache.iotdb.it.env.cluster.EnvUtils.getTimeForLogDirectory; -public class AINodeWrapper implements BaseNodeWrapper { +public class AINodeWrapper extends AbstractNodeWrapper { private static final Logger logger = IoTDBTestLogger.logger; private final String testClassName; private final String testMethodName; - private final String nodeAddress; - private Process instance; - private final int nodePort; private final long startTime; private final String seedConfigNode; @@ -73,77 +66,30 @@ public class AINodeWrapper implements BaseNodeWrapper { prepareProcess.waitFor(); } - public MLNodeWrapper( + public AINodeWrapper( String seedConfigNode, String testClassName, String testMethodName, + int clusterIndex, int[] port, long startTime) { + super(testClassName, testMethodName, port, clusterIndex, false, startTime); this.seedConfigNode = seedConfigNode; this.testClassName = testClassName; this.testMethodName = testMethodName; - this.nodeAddress = "127.0.0.1"; - this.nodePort = port[0]; this.startTime = startTime; } - @Override - public void createNodeDir() { - // Copy templateNodePath to nodePath - String destPath = getNodePath(); - try { - try { - if (new File(destPath).exists()) { - PathUtils.deleteDirectory(Paths.get(destPath)); - } - } catch (NoSuchFileException e) { - // ignored - } - // Here we need to copy without follow symbolic links, so we can't use FileUtils directly. - try (Stream<Path> s = Files.walk(Paths.get(TEMPLATE_NODE_PATH))) { - s.forEach( - source -> { - Path destination = - Paths.get(destPath, source.toString().substring(TEMPLATE_NODE_PATH.length())); - try { - Files.copy( - source, - destination, - LinkOption.NOFOLLOW_LINKS, - StandardCopyOption.COPY_ATTRIBUTES); - } catch (IOException e) { - logger.error("Got error copying files to node dest dir", e); - throw new RuntimeException(e); - } - }); - } - } catch (IOException ex) { - logger.error("Copy node dir failed", ex); - throw new AssertionError(); - } - } - - @Override - public void createLogDir() { - try { - // Make sure the log dir exist, as the first file is output by starting script directly. - FileUtils.createParentDirectories(new File(getLogPath())); - } catch (IOException ex) { - logger.error("Copy node dir failed", ex); - throw new AssertionError(); - } - } - private String getLogPath() { return getLogDirPath() + File.separator + getId() + ".log"; } - private String getLogDirPath() { + public String getLogDirPath() { return System.getProperty(USER_DIR) + File.separator + TARGET + File.separator - + "mlnode-logs" + + "ainode-logs" + File.separator + getTestLogDirName() + File.separator @@ -157,27 +103,6 @@ public class AINodeWrapper implements BaseNodeWrapper { return testClassName + "_" + testMethodName; } - @Override - public void destroyDir() { - Exception lastException = null; - for (int i = 0; i < 10; i++) { - try { - PathUtils.deleteDirectory(Paths.get(getNodePath())); - return; - } catch (IOException ex) { - lastException = ex; - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError("Delete node dir failed. " + e); - } - } - } - logger.error(lastException.getMessage()); - throw new AssertionError("Delete node dir failed."); - } - @Override public void start() { try { @@ -187,18 +112,18 @@ public class AINodeWrapper implements BaseNodeWrapper { + File.separator + TARGET + File.separator - + ML_NODE_NAME + + AI_NODE_NAME + getPort(); String propertiesFile = filePrefix + File.separator + CONFIG_PATH + File.separator + PROPERTIES_FILE; // set attribute replaceAttribute( - "mln_target_config_node_list", this.seedConfigNode, propertiesFile, stdoutFile); + "ain_target_config_node_list", this.seedConfigNode, propertiesFile, stdoutFile); replaceAttribute( - "mln_inference_rpc_port", Integer.toString(getPort()), propertiesFile, stdoutFile); + "ain_inference_rpc_port", Integer.toString(getPort()), propertiesFile, stdoutFile); - // start MLNode + // start AINode List<String> start_command = new ArrayList<>(); start_command.add(SHELL_COMMAND); start_command.add(filePrefix + File.separator + SCRIPT_PATH + File.separator + SCRIPT_FILE); @@ -213,58 +138,55 @@ public class AINodeWrapper implements BaseNodeWrapper { this.instance = processBuilder.start(); logger.info("In test {} {} started.", getTestLogDirName(), getId()); } catch (IOException | InterruptedException e) { - throw new AssertionError("Start ML Node failed. " + e + Paths.get("")); + throw new AssertionError("Start AI Node failed. " + e + Paths.get("")); } } @Override - public void stop() { - if (this.instance == null) { - return; - } - this.instance.destroy(); - try { - if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { - this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("Waiting ML Node to shutdown error. %s", e); - } + public int getMetricPort() { + // no metric currently + return -1; } @Override - public String getIp() { - return this.nodeAddress; + public String getId() { + return AI_NODE_NAME + getPort(); } - @Override - public int getPort() { - return this.nodePort; - } + /* Abstract methods, which must be implemented in ConfigNode and DataNode. */ + public void reloadMutableFields() {} + ; - @Override - public int getMetricPort() { - // no metric currently - return -1; + public void renameFile() {} + ; + + public String getSystemConfigPath() { + return ""; } + ; - @Override - public String getId() { - return ML_NODE_NAME + getPort(); + /** Return the node config file path specified through system variable */ + public String getDefaultNodeConfigPath() { + return ""; } + ; - @Override - public String getIpAndPortString() { - return this.getIp() + ":" + this.getPort(); + /** Return the common config file path specified through system variable */ + public String getDefaultCommonConfigPath() { + return ""; } + ; - @Override - public void dumpJVMSnapshot(String testCaseName) { - // there is no JVM to dump for MLNode + public void addStartCmdParams(List<String> params) {} + ; + + public String getSystemPropertiesPath() { + return ""; } + ; - private String getNodePath() { - return System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + getId(); + public MppJVMConfig initVMConfig() { + throw new UnsupportedOperationException("AINode doesn't have JVM Config"); } + ; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 942b6a9e012..e6a26bd08db 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -125,11 +125,11 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { protected final MppJVMConfig jvmConfig; protected final int clusterIndex; protected final boolean isMultiCluster; - private Process instance; + protected Process instance; private final String nodeAddress; private int nodePort; - private int metricPort; - private long startTime; + private final int metricPort; + private final long startTime; private List<String> killPoints = new ArrayList<>(); /** @@ -523,7 +523,7 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { } @Override - public final int getMetricPort() { + public int getMetricPort() { return this.metricPort; } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MLClusterIT.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java similarity index 100% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MLClusterIT.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/mlnode/it/MLEnvIT.java b/integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java similarity index 100% rename from integration-test/src/test/java/org/apache/iotdb/mlnode/it/MLEnvIT.java rename to integration-test/src/test/java/org/apache/iotdb/mlnode/it/AIEnvIT.java
