CRZbulabula commented on code in PR #17609:
URL: https://github.com/apache/iotdb/pull/17609#discussion_r4291714840
##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java:
##########
@@ -579,60 +602,84 @@ public boolean isReadOnly() {
return CommonDescriptor.getInstance().getConfig().isReadOnly();
}
- private void writeLogForSimpleConsensus(ConfigPhysicalPlan plan) {
- if (simpleLogFile.length() > LOG_FILE_MAX_SIZE) {
- try {
- simpleLogWriter.force();
- File completedFilePath = new File(FILE_PATH + startIndex + "_" +
endIndex);
- Files.move(
- simpleLogFile.toPath(), completedFilePath.toPath(),
StandardCopyOption.ATOMIC_MOVE);
- } catch (IOException e) {
- LOGGER.error(
-
ConfigNodeMessages.CAN_T_FORCE_LOGWRITER_FOR_CONFIGNODE_SIMPLECONSENSUS_MODE,
e);
+ private SimpleConsensusPersistResult
persistPlanForSimpleConsensus(ConfigPhysicalPlan plan) {
+ File persistedLogFile = null;
+ long logFileSizeBeforeWrite = 0;
+ int endIndexBeforeWrite = endIndex;
+ try {
+ if (simpleLogWriter == null || simpleLogFile == null) {
+ throw new
IOException(ConfigNodeMessages.SIMPLECONSENSUS_LOG_WRITER_IS_NOT_INITIALIZED);
}
- for (int retry = 0; retry < 5; retry++) {
- try {
- simpleLogWriter.close();
- } catch (IOException e) {
- LOGGER.warn(
- ConfigNodeMessages
-
.LOG_CONFIGREGION_UPDATE_FAILED_TO_CLOSE_THE_CONFIGNODE_SIMPLECONSENSUS_LOG_FILEPATH_ARG_RETRY_ARG_188A702E,
- simpleLogFile.getAbsolutePath(),
- retry);
- try {
- // Sleep 1s and retry
- TimeUnit.SECONDS.sleep(1);
- } catch (InterruptedException e2) {
- Thread.currentThread().interrupt();
- LOGGER.warn(
-
ConfigNodeMessages.UNEXPECTED_INTERRUPTION_DURING_THE_CLOSE_METHOD_OF_LOGWRITER);
- }
- continue;
- }
- break;
+
+ if (simpleLogFile.length() > LOG_FILE_MAX_SIZE) {
+ rollSimpleConsensusLogFile();
}
- startIndex = endIndex + 1;
- createLogFile(startIndex);
- }
- try {
+ persistedLogFile = simpleLogFile;
+ logFileSizeBeforeWrite = persistedLogFile.length();
+ endIndexBeforeWrite = endIndex;
+
ByteBuffer buffer = plan.serializeToByteBuffer();
buffer.position(buffer.limit());
simpleLogWriter.write(buffer);
+ simpleLogWriter.force();
endIndex = endIndex + 1;
} catch (Exception e) {
LOGGER.error(
ConfigNodeMessages
-
.CAN_T_SERIALIZE_CURRENT_CONFIGPHYSICALPLAN_FOR_CONFIGNODE_SIMPLECONSENSUS_MODE,
+
.PERSIST_CURRENT_CONFIGPHYSICALPLAN_FOR_CONFIGNODE_SIMPLECONSENSUS_MODE_FAILED,
e);
+ return SimpleConsensusPersistResult.failure(
Review Comment:
**[P1] Remove the appended WAL record when persistence fails**
- If `simpleLogWriter.write(buffer)` succeeds but `force()` throws, this
catch returns a failure without truncating the record already appended to the
WAL.
- `write()` then returns before calling the executor, but restart recovery
replays the record, so an operation rejected to the caller takes effect after
restart. A partial append can also leave an invalid record ahead of subsequent
writes.
- I reproduced the complete-record case with an injected `force()` failure:
status `301`, zero executor calls, and a 36-byte WAL record that
`SingleFileLogReader` successfully replays as `UpdateVersionInfo`.
Please restore and force-sync the saved pre-write offset on persistence
failure, and prevent further writes if that cleanup cannot complete. Add
coverage for append/force failures as well as executor failures.
##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java:
##########
@@ -1385,6 +1412,144 @@ public TDataNodeLocation
getLowestLoadDataNode(Set<Integer> nodes) {
return getRegisteredDataNode(dataNodeId).getLocation();
}
+ private TSStatus writeConfigPhysicalPlan(ConfigPhysicalPlan plan) {
+ try {
+ return getConsensusManager().write(plan);
+ } catch (ConsensusException e) {
+ LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
+ .setMessage(e.getMessage());
+ }
+ }
+
+ private boolean isConsensusWriteSuccessful(TSStatus status) {
+ return status != null && status.getCode() ==
TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ }
+
+ private TSStatus rollbackDataNodeRegistration(
+ TDataNodeLocation dataNodeLocation, TSStatus versionUpdateStatus) {
+ final TSStatus rollbackStatus =
+ writeConfigPhysicalPlan(
+ new
RemoveDataNodePlan(Collections.singletonList(dataNodeLocation)));
+ final String failureMessage =
+ String.format(
+ "Failed to persist version info for DataNode %d: %s",
+ dataNodeLocation.getDataNodeId(),
describeStatus(versionUpdateStatus));
+ if (isConsensusWriteSuccessful(rollbackStatus)) {
+ return buildStatus(
+ versionUpdateStatus.getCode(),
+ failureMessage,
+ "The registration has been rolled back. Please retry the
registration.");
+ }
+
+ LOGGER.error(
+ "Failed to roll back DataNode registration {} after version info
persistence failure. "
+ + ROLLBACK_STATUS_LOG_FORMAT,
+ dataNodeLocation,
+ versionUpdateStatus,
+ rollbackStatus);
+ return buildStatus(
+ rollbackStatus.getCode(),
+ failureMessage,
+ String.format(REGISTRATION_ROLLBACK_FAILED_FORMAT,
describeStatus(rollbackStatus)),
+ MANUAL_REGISTRATION_CLEANUP_MESSAGE);
+ }
+
+ private TSStatus rollbackAiNodeRegistration(
+ TAINodeLocation aiNodeLocation, TSStatus versionUpdateStatus) {
+ final TSStatus rollbackStatus = writeConfigPhysicalPlan(new
RemoveAINodePlan(aiNodeLocation));
+ final String failureMessage =
+ String.format(
+ "Failed to persist version info for AINode %d: %s",
+ aiNodeLocation.getAiNodeId(), describeStatus(versionUpdateStatus));
+ if (isConsensusWriteSuccessful(rollbackStatus)) {
+ return buildStatus(
+ versionUpdateStatus.getCode(),
+ failureMessage,
+ "The registration has been rolled back. Please retry the
registration.");
+ }
+
+ LOGGER.error(
+ "Failed to roll back AINode registration {} after version info
persistence failure. "
+ + ROLLBACK_STATUS_LOG_FORMAT,
+ aiNodeLocation,
+ versionUpdateStatus,
+ rollbackStatus);
+ return buildStatus(
+ rollbackStatus.getCode(),
+ failureMessage,
+ String.format(REGISTRATION_ROLLBACK_FAILED_FORMAT,
describeStatus(rollbackStatus)),
+ MANUAL_REGISTRATION_CLEANUP_MESSAGE);
+ }
+
+ private TSStatus rollbackConfigNodeRegistration(
+ TConfigNodeLocation configNodeLocation, TSStatus versionUpdateStatus) {
+ final TSStatus rollbackStatus =
+ writeConfigPhysicalPlan(new RemoveConfigNodePlan(configNodeLocation));
Review Comment:
**[P1] Avoid decommissioning the joining ConfigNode during registration
rollback**
- At this point the joining ConfigNode is already a Raft member:
`AddConfigNodeProcedure` completes `ADD_PEER` before entering
`REGISTER_SUCCESS`. It therefore also applies this `RemoveConfigNodePlan`.
- `NodeInfo.removeConfigNode()` sets the local node status to `Removing`
when the removed ID is its own. The procedure retries `REGISTER_SUCCESS` after
this exception, but neither `applyConfigNode()` nor `notifyRegisterSuccess()`
resets that status.
- A transient version-write failure followed by a successful retry
consequently leaves the newly registered ConfigNode rejecting requests in
`ConfigManager.confirmLeader()`.
Please use a registration rollback that avoids node-removal lifecycle side
effects, or explicitly restore the appropriate node status when registration
succeeds. A regression test should cover version-write failure followed by a
successful registration retry.
##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java:
##########
@@ -1385,6 +1412,144 @@ public TDataNodeLocation
getLowestLoadDataNode(Set<Integer> nodes) {
return getRegisteredDataNode(dataNodeId).getLocation();
}
+ private TSStatus writeConfigPhysicalPlan(ConfigPhysicalPlan plan) {
+ try {
+ return getConsensusManager().write(plan);
+ } catch (ConsensusException e) {
+ LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
+ .setMessage(e.getMessage());
+ }
+ }
+
+ private boolean isConsensusWriteSuccessful(TSStatus status) {
+ return status != null && status.getCode() ==
TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ }
+
+ private TSStatus rollbackDataNodeRegistration(
+ TDataNodeLocation dataNodeLocation, TSStatus versionUpdateStatus) {
+ final TSStatus rollbackStatus =
+ writeConfigPhysicalPlan(
+ new
RemoveDataNodePlan(Collections.singletonList(dataNodeLocation)));
+ final String failureMessage =
+ String.format(
+ "Failed to persist version info for DataNode %d: %s",
+ dataNodeLocation.getDataNodeId(),
describeStatus(versionUpdateStatus));
Review Comment:
**[P2] Localize the new registration failure templates**
- This template, the rollback result messages/logs below, and the analogous
AINode/ConfigNode helpers introduce raw English strings in Java code. The new
exception/status descriptions and the constants near the top of this class have
the same issue.
- These strings remain English under `-P with-zh-locale`, because that build
only swaps the locale-specific message classes.
Please put each complete user-visible or operator-visible template in both
the English and Chinese `ConfigNodeMessages` classes, preserve its format
placeholders, and reference those constants here. This also applies to the new
rollback log and exception templates.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]