This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch rel/1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.0 by this push:
new 3bfce91d6d [To rel/1.0][IOTDB-5201][IOTDB-5223] Extract DataNode
getConfig and add check for cross-disk directories (#8471)
3bfce91d6d is described below
commit 3bfce91d6df5876dbbbcf09c21270f27e63d5a06
Author: Chen YZ <[email protected]>
AuthorDate: Thu Dec 15 16:01:09 2022 +0800
[To rel/1.0][IOTDB-5201][IOTDB-5223] Extract DataNode getConfig and add
check for cross-disk directories (#8471)
* [IOTDB-5201] Extracting getConfig from the registerDataNode interface
(#8441)
* [IOTDB-5223] Check data directories cross-disk under RatisConsensus
(#8469)
* done
* enhance loop
---
.../consensus/response/ConfigurationResp.java | 66 +++++++++++++++++
.../consensus/response/DataNodeRegisterResp.java | 23 ------
.../iotdb/confignode/manager/ConfigManager.java | 14 ++++
.../apache/iotdb/confignode/manager/IManager.java | 7 ++
.../iotdb/confignode/manager/node/NodeManager.java | 21 ++++--
.../thrift/ConfigNodeRPCServiceProcessor.java | 12 ++++
.../apache/iotdb/db/client/ConfigNodeClient.java | 22 ++++++
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 8 +++
.../db/conf/directories/DirectoryChecker.java | 27 +++++++
.../java/org/apache/iotdb/db/service/DataNode.java | 82 ++++++++++++++--------
.../db/service/DataNodeServerCommandLine.java | 2 +-
.../src/main/thrift/confignode.thrift | 23 ++++--
12 files changed, 240 insertions(+), 67 deletions(-)
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/ConfigurationResp.java
b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/ConfigurationResp.java
new file mode 100644
index 0000000000..70e8af6df6
--- /dev/null
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/ConfigurationResp.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.consensus.response;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.confignode.rpc.thrift.TCQConfig;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigurationResp;
+import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
+import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig;
+import org.apache.iotdb.consensus.common.DataSet;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+public class ConfigurationResp implements DataSet {
+
+ private TSStatus status;
+ private TGlobalConfig globalConfig;
+ private TRatisConfig ratisConfig;
+ private TCQConfig cqConfig;
+
+ public TSStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(TSStatus status) {
+ this.status = status;
+ }
+
+ public void setGlobalConfig(TGlobalConfig globalConfig) {
+ this.globalConfig = globalConfig;
+ }
+
+ public void setRatisConfig(TRatisConfig ratisConfig) {
+ this.ratisConfig = ratisConfig;
+ }
+
+ public void setCqConfig(TCQConfig cqConfig) {
+ this.cqConfig = cqConfig;
+ }
+
+ public TConfigurationResp convertToRpcDataNodeRegisterResp() {
+ TConfigurationResp resp = new TConfigurationResp();
+ resp.setStatus(status);
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ resp.setGlobalConfig(globalConfig);
+ resp.setRatisConfig(ratisConfig);
+ resp.setCqConfig(cqConfig);
+ }
+ return resp;
+ }
+}
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java
b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java
index bf4e64f57c..0a90903f35 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java
@@ -20,10 +20,7 @@ package org.apache.iotdb.confignode.consensus.response;
import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.confignode.rpc.thrift.TCQConfig;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
-import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
-import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -39,10 +36,6 @@ public class DataNodeRegisterResp implements DataSet {
private TSStatus status;
private List<TConfigNodeLocation> configNodeList;
private Integer dataNodeId;
- private TGlobalConfig globalConfig;
- private TRatisConfig ratisConfig;
-
- private TCQConfig cqConfig;
private byte[] templateInfo;
private List<ByteBuffer> allTriggerInformation;
private List<ByteBuffer> allUDFInformation;
@@ -51,7 +44,6 @@ public class DataNodeRegisterResp implements DataSet {
public DataNodeRegisterResp() {
this.dataNodeId = null;
- this.globalConfig = null;
}
public TSStatus getStatus() {
@@ -70,18 +62,6 @@ public class DataNodeRegisterResp implements DataSet {
this.dataNodeId = dataNodeId;
}
- public void setGlobalConfig(TGlobalConfig globalConfig) {
- this.globalConfig = globalConfig;
- }
-
- public void setRatisConfig(TRatisConfig ratisConfig) {
- this.ratisConfig = ratisConfig;
- }
-
- public void setCqConfig(TCQConfig cqConfig) {
- this.cqConfig = cqConfig;
- }
-
public void setTemplateInfo(byte[] templateInfo) {
this.templateInfo = templateInfo;
}
@@ -120,10 +100,7 @@ public class DataNodeRegisterResp implements DataSet {
|| status.getCode() ==
TSStatusCode.DATANODE_ALREADY_REGISTERED.getStatusCode()
|| status.getCode() ==
TSStatusCode.DATANODE_NOT_EXIST.getStatusCode()) {
resp.setDataNodeId(dataNodeId);
- resp.setGlobalConfig(globalConfig);
resp.setTemplateInfo(templateInfo);
- resp.setRatisConfig(ratisConfig);
- resp.setCqConfig(cqConfig);
resp.setAllTriggerInformation(allTriggerInformation);
resp.setAllUDFInformation(allUDFInformation);
resp.setAllTTLInformation(allTTLInformation);
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 24c3b587e7..439cac9131 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -65,6 +65,7 @@ import
org.apache.iotdb.confignode.consensus.request.write.storagegroup.SetTimeP
import
org.apache.iotdb.confignode.consensus.request.write.sync.CreatePipeSinkPlan;
import
org.apache.iotdb.confignode.consensus.request.write.sync.DropPipeSinkPlan;
import
org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan;
+import org.apache.iotdb.confignode.consensus.response.ConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.CountStorageGroupResp;
import
org.apache.iotdb.confignode.consensus.response.DataNodeConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeRegisterResp;
@@ -280,6 +281,19 @@ public class ConfigManager implements IManager {
return dataSet;
}
+ @Override
+ public DataSet getConfiguration() {
+ TSStatus status = confirmLeader();
+ ConfigurationResp dataSet;
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ dataSet = (ConfigurationResp) nodeManager.getConfiguration();
+ } else {
+ dataSet = new ConfigurationResp();
+ dataSet.setStatus(status);
+ }
+ return dataSet;
+ }
+
@Override
public DataSet removeDataNode(RemoveDataNodePlan removeDataNodePlan) {
TSStatus status = confirmLeader();
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index b20a7f3c24..5079545cf0 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -178,6 +178,13 @@ public interface IManager {
*/
DataSet registerDataNode(RegisterDataNodePlan registerDataNodePlan);
+ /**
+ * Get configuration information that is not associated with the DataNodeId
+ *
+ * @return ConfigurationResp
+ */
+ DataSet getConfiguration();
+
/**
* Remove DataNode
*
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
index 0026de9710..396207114e 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
@@ -48,6 +48,7 @@ import
org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConf
import
org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
import
org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
import
org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan;
+import org.apache.iotdb.confignode.consensus.response.ConfigurationResp;
import
org.apache.iotdb.confignode.consensus.response.DataNodeConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeRegisterResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeToStatusResp;
@@ -74,6 +75,7 @@ import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.consensus.common.response.ConsensusGenericResponse;
import org.apache.iotdb.mpp.rpc.thrift.THeartbeatReq;
+import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
@@ -144,7 +146,7 @@ public class NodeManager {
this.random = new Random(System.currentTimeMillis());
}
- private void setGlobalConfig(DataNodeRegisterResp dataSet) {
+ private void setGlobalConfig(ConfigurationResp dataSet) {
// Set TGlobalConfig
final ConfigNodeConfig configNodeConfig =
ConfigNodeDescriptor.getInstance().getConf();
final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
@@ -162,7 +164,7 @@ public class NodeManager {
dataSet.setGlobalConfig(globalConfig);
}
- private void setRatisConfig(DataNodeRegisterResp dataSet) {
+ private void setRatisConfig(ConfigurationResp dataSet) {
final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
TRatisConfig ratisConfig = new TRatisConfig();
@@ -215,7 +217,7 @@ public class NodeManager {
dataSet.setRatisConfig(ratisConfig);
}
- private void setCQConfig(DataNodeRegisterResp dataSet) {
+ private void setCQConfig(ConfigurationResp dataSet) {
final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
TCQConfig cqConfig = new TCQConfig();
cqConfig.setCqMinEveryIntervalInMs(conf.getCqMinEveryIntervalInMs());
@@ -261,6 +263,17 @@ public class NodeManager {
dataSet.setDataNodeId(
registerDataNodePlan.getDataNodeConfiguration().getLocation().getDataNodeId());
dataSet.setConfigNodeList(getRegisteredConfigNodes());
+ return dataSet;
+ }
+
+ /**
+ * Get configuration
+ *
+ * @return ConfigurationResp. The TSStatus will be set to SUCCESS_STATUS.
+ */
+ public DataSet getConfiguration() {
+ ConfigurationResp dataSet = new ConfigurationResp();
+ dataSet.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS));
setGlobalConfig(dataSet);
setRatisConfig(dataSet);
setCQConfig(dataSet);
@@ -357,8 +370,6 @@ public class NodeManager {
dataSet.setStatus(status);
dataSet.setDataNodeId(updateDataNodePlan.getDataNodeLocation().getDataNodeId());
dataSet.setConfigNodeList(getRegisteredConfigNodes());
- setGlobalConfig(dataSet);
- setRatisConfig(dataSet);
return dataSet;
}
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
b/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 93a1e12164..4354b492b2 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -56,6 +56,7 @@ import
org.apache.iotdb.confignode.consensus.request.write.storagegroup.SetTTLPl
import
org.apache.iotdb.confignode.consensus.request.write.storagegroup.SetTimePartitionIntervalPlan;
import
org.apache.iotdb.confignode.consensus.request.write.sync.CreatePipeSinkPlan;
import
org.apache.iotdb.confignode.consensus.request.write.sync.DropPipeSinkPlan;
+import org.apache.iotdb.confignode.consensus.response.ConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.CountStorageGroupResp;
import
org.apache.iotdb.confignode.consensus.response.DataNodeConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeRegisterResp;
@@ -72,6 +73,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp;
import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
@@ -189,6 +191,16 @@ public class ConfigNodeRPCServiceProcessor implements
IConfigNodeRPCService.Ifac
return resp;
}
+ @Override
+ public TConfigurationResp getConfiguration() {
+ TConfigurationResp resp =
+ ((ConfigurationResp)
configManager.getConfiguration()).convertToRpcDataNodeRegisterResp();
+
+ // Print log to record the ConfigNode that performs the
GetConfigurationRequest
+ LOGGER.info("Execute GetConfigurationRequest with result {}", resp);
+ return resp;
+ }
+
@Override
public TDataNodeRemoveResp removeDataNode(TDataNodeRemoveReq req) {
LOGGER.info("ConfigNode RPC Service start to remove DataNode, req: {}",
req);
diff --git
a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
index a569857df9..3b51b56de9 100644
--- a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
+++ b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp;
import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
@@ -332,6 +333,27 @@ public class ConfigNodeClient
throw new TException(MSG_RECONNECTION_FAIL);
}
+ @Override
+ public TConfigurationResp getConfiguration() throws TException {
+ for (int i = 0; i < RETRY_NUM; i++) {
+ try {
+ TConfigurationResp resp = client.getConfiguration();
+ if (!updateConfigNodeLeader(resp.status)) {
+ return resp;
+ }
+ } catch (TException e) {
+ logger.warn(
+ "Failed to connect to ConfigNode {} from DataNode {} when
executing {}",
+ configNode,
+ config.getAddressAndPort(),
+ Thread.currentThread().getStackTrace()[1].getMethodName());
+ configLeader = null;
+ }
+ waitAndReconnect();
+ }
+ throw new TException(MSG_RECONNECTION_FAIL);
+ }
+
@Override
public TDataNodeRemoveResp removeDataNode(TDataNodeRemoveReq req) throws
TException {
for (int i = 0; i < RETRY_NUM; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
index 5cda30853c..44c82254ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
@@ -205,10 +205,18 @@ public class IoTDBStartCheck {
* accessing same director.
*/
public void checkDirectory() throws ConfigurationException, IOException {
+
// check data dirs
for (String dataDir : config.getDataDirs()) {
DirectoryChecker.getInstance().registerDirectory(new File(dataDir));
}
+ if (config.isClusterMode()
+ &&
config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS))
{
+ if (DirectoryChecker.getInstance().isCrossDisk(config.getDataDirs())) {
+ throw new ConfigurationException(
+ "Configuring the data directories as cross-disk directories is not
supported under RatisConsensus(it will be supported in a later version).");
+ }
+ }
// check system dir
DirectoryChecker.getInstance().registerDirectory(new
File(config.getSystemDir()));
// check WAL dir
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
index 9ea5bf34b5..71c021dfa2 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
@@ -31,6 +31,9 @@ import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@@ -84,6 +87,30 @@ public class DirectoryChecker {
randomAccessFileList.add(randomAccessFile);
}
+ public boolean isCrossDisk(String[] dirs) throws IOException {
+ if (dirs.length < 2) {
+ return false;
+ }
+ Path root = mountOf(new File(dirs[0]).toPath());
+ for (int i = 1; i < dirs.length; i++) {
+ Path path = mountOf(new File(dirs[i]).toPath());
+ if (!path.equals(root)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Path mountOf(Path p) throws IOException {
+ FileStore fs = Files.getFileStore(p);
+ Path temp = p.toAbsolutePath();
+ Path mountp = temp;
+ while ((temp = temp.getParent()) != null &&
fs.equals(Files.getFileStore(temp))) {
+ mountp = temp;
+ }
+ return mountp;
+ }
+
public void deregisterAll() {
try {
for (RandomAccessFile randomAccessFile : randomAccessFileList) {
diff --git a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
index 8f4989f3d0..060f432b1a 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.commons.udf.UDFInformation;
import org.apache.iotdb.commons.udf.service.UDFClassLoaderManager;
import org.apache.iotdb.commons.udf.service.UDFExecutableManager;
import org.apache.iotdb.commons.udf.service.UDFManagementService;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
@@ -91,8 +92,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
-import static org.apache.iotdb.db.service.DataNodeServerCommandLine.MODE_START;
-
public class DataNode implements DataNodeMBean {
private static final Logger logger = LoggerFactory.getLogger(DataNode.class);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
@@ -102,10 +101,10 @@ public class DataNode implements DataNodeMBean {
"%s:%s=%s", "org.apache.iotdb.datanode.service",
IoTDBConstant.JMX_TYPE, "DataNode");
/**
- * when joining a cluster this node will retry at most "DEFAULT_JOIN_RETRY"
times before returning
- * a failure to the client
+ * when joining a cluster or getting configuration this node will retry at
most "DEFAULT_RETRY"
+ * times before returning a failure to the client
*/
- private static final int DEFAULT_JOIN_RETRY = 10;
+ private static final int DEFAULT_RETRY = 10;
private final TEndPoint thisNode = new TEndPoint();
@@ -132,13 +131,9 @@ public class DataNode implements DataNodeMBean {
new DataNodeServerCommandLine().doMain(args);
}
- protected void serverCheckAndInit(String mode) throws
ConfigurationException, IOException {
+ protected void serverCheckAndInit() throws ConfigurationException,
IOException {
config.setClusterMode(true);
IoTDBStartCheck.getInstance().checkConfig();
- if (MODE_START.equals(mode)) {
- // Only checkDirectory when start DataNode
- IoTDBStartCheck.getInstance().checkDirectory();
- }
// TODO: check configuration for data node
for (TEndPoint endPoint : config.getTargetConfigNodeList()) {
@@ -156,6 +151,8 @@ public class DataNode implements DataNodeMBean {
try {
// prepare cluster IoTDB-DataNode
prepareDataNode();
+ // pull and check configuration from ConfigNode
+ pullAndCheckConfiguration();
// register current DataNode to ConfigNode
registerInConfigNode();
// active DataNode
@@ -198,9 +195,51 @@ public class DataNode implements DataNodeMBean {
JMXService.registerMBean(getInstance(), mbeanName);
}
+ private void pullAndCheckConfiguration() throws StartupException {
+ int retry = DEFAULT_RETRY;
+
+
ConfigNodeInfo.getInstance().updateConfigNodeList(config.getTargetConfigNodeList());
+ // get and check configuration
+ while (retry > 0) {
+ try (ConfigNodeClient configNodeClient = new ConfigNodeClient()) {
+ TConfigurationResp configuration = configNodeClient.getConfiguration();
+
IoTDBDescriptor.getInstance().loadGlobalConfig(configuration.globalConfig);
+
IoTDBDescriptor.getInstance().loadRatisConfig(configuration.ratisConfig);
+ IoTDBDescriptor.getInstance().loadCQConfig(configuration.cqConfig);
+
+
CommonDescriptor.getInstance().loadGlobalConfig(configuration.globalConfig);
+ if (!IoTDBStartCheck.getInstance()
+ .checkConsensusProtocolExists(TConsensusGroupType.DataRegion)) {
+ config.setDataRegionConsensusProtocolClass(
+
configuration.globalConfig.getDataRegionConsensusProtocolClass());
+ }
+
+ if (!IoTDBStartCheck.getInstance()
+ .checkConsensusProtocolExists(TConsensusGroupType.SchemaRegion)) {
+ config.setSchemaRegionConsensusProtocolClass(
+
configuration.globalConfig.getSchemaRegionConsensusProtocolClass());
+ }
+
+ IoTDBStartCheck.getInstance().checkDirectory();
+
IoTDBStartCheck.getInstance().serializeGlobalConfig(configuration.globalConfig);
+ return;
+ } catch (TException e) {
+ // read config nodes from system.properties
+ logger.warn("Cannot register to the cluster, because: {}",
e.getMessage());
+ ConfigNodeInfo.getInstance().loadConfigNodeList();
+ } catch (Exception e) {
+ throw new StartupException(e.getMessage());
+ }
+ retry--;
+ }
+ // all tries failed
+ logger.error("Cannot get configuration from ConfigNode after {} retries",
DEFAULT_RETRY);
+ throw new StartupException("Cannot get configuration from ConfigNode");
+ }
+
/** register DataNode with ConfigNode */
private void registerInConfigNode() throws StartupException {
- int retry = DEFAULT_JOIN_RETRY;
+ int retry = DEFAULT_RETRY;
ConfigNodeInfo.getInstance().updateConfigNodeList(config.getTargetConfigNodeList());
while (retry > 0) {
@@ -238,25 +277,8 @@ public class DataNode implements DataNodeMBean {
IoTDBStartCheck.getInstance().serializeDataNodeId(dataNodeID);
config.setDataNodeId(dataNodeID);
}
-
IoTDBDescriptor.getInstance().loadGlobalConfig(dataNodeRegisterResp.globalConfig);
-
IoTDBDescriptor.getInstance().loadRatisConfig(dataNodeRegisterResp.ratisConfig);
-
IoTDBDescriptor.getInstance().loadCQConfig(dataNodeRegisterResp.cqConfig);
IoTDBDescriptor.getInstance().initClusterSchemaMemoryAllocate();
-
CommonDescriptor.getInstance().loadGlobalConfig(dataNodeRegisterResp.globalConfig);
-
- if (!IoTDBStartCheck.getInstance()
- .checkConsensusProtocolExists(TConsensusGroupType.DataRegion)) {
- config.setDataRegionConsensusProtocolClass(
-
dataNodeRegisterResp.globalConfig.getDataRegionConsensusProtocolClass());
- }
-
- if (!IoTDBStartCheck.getInstance()
- .checkConsensusProtocolExists(TConsensusGroupType.SchemaRegion))
{
- config.setSchemaRegionConsensusProtocolClass(
-
dataNodeRegisterResp.globalConfig.getSchemaRegionConsensusProtocolClass());
- }
-
// In current implementation, only IoTConsensus need separated
memory from Consensus
if (!config
.getDataRegionConsensusProtocolClass()
@@ -264,8 +286,6 @@ public class DataNode implements DataNodeMBean {
IoTDBDescriptor.getInstance().reclaimConsensusMemory();
}
-
IoTDBStartCheck.getInstance().serializeGlobalConfig(dataNodeRegisterResp.globalConfig);
-
logger.info("Register to the cluster successfully");
return;
} else if (dataNodeRegisterResp.getStatus().getCode()
@@ -294,7 +314,7 @@ public class DataNode implements DataNodeMBean {
retry--;
}
// all tries failed
- logger.error("Cannot register to the cluster after {} retries",
DEFAULT_JOIN_RETRY);
+ logger.error("Cannot register to the cluster after {} retries",
DEFAULT_RETRY);
throw new StartupException("Cannot register to the cluster.");
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/DataNodeServerCommandLine.java
b/server/src/main/java/org/apache/iotdb/db/service/DataNodeServerCommandLine.java
index 55bba9fb9c..0ded2a8bcd 100644
---
a/server/src/main/java/org/apache/iotdb/db/service/DataNodeServerCommandLine.java
+++
b/server/src/main/java/org/apache/iotdb/db/service/DataNodeServerCommandLine.java
@@ -78,7 +78,7 @@ public class DataNodeServerCommandLine extends
ServerCommandLine {
// Check config of IoTDB, and set some configs in cluster mode
try {
- dataNode.serverCheckAndInit(mode);
+ dataNode.serverCheckAndInit();
} catch (ConfigurationException | IOException e) {
LOGGER.error("Meet error when doing start checking", e);
return -1;
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift
b/thrift-confignode/src/main/thrift/confignode.thrift
index a9ab4c31c9..d13ae7bad6 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -33,13 +33,17 @@ struct TDataNodeRegisterResp {
1: required common.TSStatus status
2: required list<common.TConfigNodeLocation> configNodeList
3: optional i32 dataNodeId
- 4: optional TGlobalConfig globalConfig
- 5: optional binary templateInfo
- 6: optional TRatisConfig ratisConfig
- 7: optional list<binary> allTriggerInformation
- 8: optional TCQConfig cqConfig
- 9: optional list<binary> allUDFInformation
- 10: optional binary allTTLInformation
+ 4: optional binary templateInfo
+ 5: optional list<binary> allTriggerInformation
+ 6: optional list<binary> allUDFInformation
+ 7: optional binary allTTLInformation
+}
+
+struct TConfigurationResp{
+ 1: required common.TSStatus status
+ 2: optional TGlobalConfig globalConfig
+ 3: optional TRatisConfig ratisConfig
+ 4: optional TCQConfig cqConfig
}
struct TGlobalConfig {
@@ -632,6 +636,11 @@ service IConfigNodeRPCService {
*/
TDataNodeRegisterResp registerDataNode(TDataNodeRegisterReq req)
+ /**
+ * Get configuration information that is not associated with the DataNodeId
+ */
+ TConfigurationResp getConfiguration()
+
/**
* Generate a set of DataNodeRemoveProcedure to remove some specific
DataNodes from the cluster
*
