JackieTien97 commented on code in PR #18939:
URL: https://github.com/apache/iotdb/pull/18939#discussion_r4324448937
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java:
##########
@@ -166,29 +190,83 @@ public ReporterType getReporterType() {
@Override
protected void writeMetricToIoTDB(Map<String, Object> valueMap, String
prefix, long time) {
- service.execute(
- () -> {
- try {
- TSStatus result = insertRecord(valueMap, prefix, time);
- if (result.getCode() ==
TSStatusCode.METADATA_ERROR.getStatusCode()) {
- createTimeSeries(valueMap, prefix);
- result = insertRecord(valueMap, prefix, time);
- }
- if (result.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- LOGGER.warn(DataNodeMiscMessages.FAILED_UPDATE_METRIC_VALUE,
result);
- }
- } catch (IoTDBConnectionException e1) {
- LOGGER.warn(
- DataNodeMiscMessages
-
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_CONNECTION_B0FC4929,
- e1);
- } catch (IllegalPathException | QueryProcessException e2) {
- LOGGER.warn(
- DataNodeMiscMessages
-
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_INTERNAL_E5C64806,
- e2);
- }
- });
+ if (isStopping()) {
+ return;
+ }
+ try {
+ service.execute(() -> writeMetric(valueMap, prefix, time));
+ } catch (RejectedExecutionException e) {
+ // stop() can shut down the executor between the check above and
submission.
+ if (!stopped) {
+ throw e;
+ }
+ }
+ }
+
+ private void writeMetric(Map<String, Object> valueMap, String prefix, long
time) {
+ if (isStopping()) {
+ return;
+ }
+ try {
+ TSStatus result = insertRecord(valueMap, prefix, time);
+ if (result.getCode() == TSStatusCode.METADATA_ERROR.getStatusCode() &&
!isStopping()) {
+ createTimeSeries(valueMap, prefix);
+ if (isStopping()) {
+ return;
+ }
+ result = insertRecord(valueMap, prefix, time);
+ }
+ if (result.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ if (isExpectedShutdownFailure(result)) {
+ LOGGER.debug(DataNodeMiscMessages.FAILED_UPDATE_METRIC_VALUE,
result);
+ } else {
+ LOGGER.warn(DataNodeMiscMessages.FAILED_UPDATE_METRIC_VALUE, result);
+ }
+ }
+ } catch (IoTDBConnectionException e1) {
+ if (isInterruptedDuringShutdown(e1)) {
+ LOGGER.debug(
+ DataNodeMiscMessages
+
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_CONNECTION_B0FC4929,
+ e1);
+ } else {
+ LOGGER.warn(
+ DataNodeMiscMessages
+
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_CONNECTION_B0FC4929,
+ e1);
+ }
+ } catch (IllegalPathException | QueryProcessException e2) {
+ if (isInterruptedDuringShutdown(e2)) {
+ LOGGER.debug(
+ DataNodeMiscMessages
+
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_INTERNAL_E5C64806,
+ e2);
+ } else {
+ LOGGER.warn(
+ DataNodeMiscMessages
+
.MISC_LOG_FAILED_TO_UPDATE_THE_VALUE_OF_METRIC_BECAUSE_OF_INTERNAL_E5C64806,
+ e2);
+ }
+ }
+ }
+
+ private boolean isStopping() {
+ return stopped || CommonDescriptor.getInstance().getConfig().isStopping();
+ }
+
+ private boolean isExpectedShutdownFailure(TSStatus status) {
+ return isStopping()
+ && (Thread.currentThread().isInterrupted()
+ || (CommonDescriptor.getInstance().getConfig().isStopping()
+ && status.getCode() ==
TSStatusCode.SYSTEM_READ_ONLY.getStatusCode()));
Review Comment:
[P2] Preserve warnings for existing failures during reporter reload
When an in-flight write has already recorded a `SYSTEM_READ_ONLY` or
`WRITE_PROCESS_ERROR` failure and the reporter is reloaded, `stop()` calls
`shutdownNow()` and interrupts the worker. `QueryExecution.getStatus()` can
still return that pre-existing `failureStatus` from its `InterruptedException`
handler (lines 644-652), so the interrupt flag does not establish that the
returned error was caused by shutdown. This unconditional `isInterrupted()`
branch then downgrades the real failure to DEBUG, even when the node is not
stopping; it also bypasses the node-shutdown restriction on `SYSTEM_READ_ONLY`
below.
I reproduced both cases with a real scheduled executor and the actual
`QueryExecution.getStatus()`: status codes 600 and 605 were retained, but both
metric-write messages were DEBUG. The existing
`testReporterReloadDoesNotHideReadOnlyFailure` misses this because its mocked
executor does not actually interrupt the worker on `shutdownNow()`.
Please retain WARN for identifiable non-shutdown failures and restrict
interruption-based suppression to results attributable to interruption. Add
coverage using a real executor for this response/reload race.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]