This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 0517abfe8c3 Pipe: improve progress coverage checks (#17958)
0517abfe8c3 is described below
commit 0517abfe8c3d80d4b3885f5927d183b28de51db2
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 14:26:35 2026 +0800
Pipe: improve progress coverage checks (#17958)
* Pipe: improve progress coverage checks (#17940)
* Pipe: improve progress coverage checks
* Pipe: address shutdown progress review comments
* Pipe: refine Chinese shutdown progress messages
(cherry picked from commit b33278688c2fc5f8c6f860253204b4e835f16487)
* Fix historical TsFile selection statistics
* Fix missing time window progress helper
---
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 20 +++++----
.../PipeHistoricalDataRegionTsFileSource.java | 48 ++++++++++++++++++++--
.../iotdb/db/service/DataNodeShutdownHook.java | 2 +-
.../PipeHistoricalDataRegionTsFileSourceTest.java | 39 +++++++++++++++---
.../PipeTsFileEpochProgressIndexKeeperTest.java | 15 +++++--
5 files changed, 103 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index 19391952917..9021253940d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -765,7 +765,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
persistThread.setDaemon(true);
LOGGER.info(
- "Start to persist all pipe progress indexes during shutdown, pipe
count {}, timeout {} ms.",
+ "Start to persist all pipe progress indexes during shutdown, pipe
count: {}, timeout: {} ms.",
getPipeCount(),
normalizedTimeoutInMs);
persistThread.start();
@@ -816,7 +816,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
.mapToInt(ByteBuffer::remaining)
.sum();
LOGGER.info(
- "Collected pipe metas for shutdown progress persist, pipe count {},
pipe meta count {}, pipe meta size {} bytes, cost {} ms.",
+ "Collected pipe metas for shutdown progress persist, local pipe
count: {}, "
+ + "collected pipe meta count: {}, total serialized size: {}
bytes, cost: {} ms.",
pipeCount,
pipeMetaCount,
pipeMetaSizeInBytes,
@@ -825,7 +826,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
if (resp.getPipeMetaList().isEmpty()) {
if (pipeCount != 0) {
LOGGER.info(
- "Collected empty pipe metas during shutdown while pipe count is
{}.", pipeCount);
+ "Collected empty pipe metas during shutdown while local pipe
count is {}.",
+ pipeCount);
return false;
}
return true;
@@ -834,7 +836,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
try (final ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID))
{
LOGGER.info(
- "Start to push heartbeat shutdown pipe meta to ConfigNode, data
node id {}, pipe count {}, pipe meta count {}, pipe meta size {} bytes.",
+ "Start to push shutdown pipe meta heartbeat to ConfigNode,
dataNodeId: {}, "
+ + "local pipe count: {}, collected pipe meta count: {}, total
serialized size: {} bytes.",
IoTDBDescriptor.getInstance().getConfig().getDataNodeId(),
pipeCount,
pipeMetaCount,
@@ -845,20 +848,21 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
IoTDBDescriptor.getInstance().getConfig().getDataNodeId(),
resp);
final long pushCostTime = System.currentTimeMillis() - pushStartTime;
if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != result.getCode()) {
- LOGGER.warn("Failed to persist progress index to ConfigNode, status:
{}", result);
+ LOGGER.warn("Failed to persist progress index to configNode, status:
{}", result);
LOGGER.warn(
- "Failed to push heartbeat shutdown pipe meta to ConfigNode,
status {}, cost {} ms.",
+ "Failed to push shutdown pipe meta heartbeat to ConfigNode,
status: {}, cost {} ms.",
result,
pushCostTime);
return false;
} else {
LOGGER.info(
- "Successfully finished pushing heartbeat shutdown pipe meta to
ConfigNode, pipe count {}, pipe meta count {}, pipe meta size {} bytes, cost {}
ms.",
+ "Successfully pushed shutdown pipe meta heartbeat to ConfigNode,
local pipe count: {}, "
+ + "collected pipe meta count: {}, total serialized size: {}
bytes, cost {} ms.",
pipeCount,
pipeMetaCount,
pipeMetaSizeInBytes,
pushCostTime);
- LOGGER.info("Successfully persisted all pipe's info to ConfigNode.");
+ LOGGER.info("Successfully persisted all pipe's info to configNode.");
return true;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
index 8fff0e77609..d3ec952d97a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
@@ -413,15 +413,17 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
originalUnSequenceTsFileCount,
startIndex);
+ final HistoricalTsFileExtractionStatistics statistics =
+ new HistoricalTsFileExtractionStatistics();
final Collection<TsFileResource> sequenceTsFileResources =
tsFileManager.getTsFileList(true).stream()
.peek(originalResourceList::add)
- .filter(this::shouldExtractTsFileResource)
+ .filter(resource -> shouldExtractTsFileResource(resource,
statistics))
.collect(Collectors.toList());
final Collection<TsFileResource> unSequenceTsFileResources =
tsFileManager.getTsFileList(false).stream()
.peek(originalResourceList::add)
- .filter(this::shouldExtractTsFileResource)
+ .filter(resource -> shouldExtractTsFileResource(resource,
statistics))
.collect(Collectors.toList());
filteredTsFileResources.addAll(sequenceTsFileResources);
filteredTsFileResources.addAll(unSequenceTsFileResources);
@@ -437,6 +439,7 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
}
return false;
} catch (final IOException e) {
+ ++statistics.pinFailedCount;
LOGGER.warn("Pipe: failed to pin TsFileResource {}",
resource.getTsFilePath(), e);
return true;
}
@@ -465,6 +468,22 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
filteredTsFileResources.size(),
originalSequenceTsFileCount + originalUnSequenceTsFileCount,
System.currentTimeMillis() - startHistoricalExtractionTime);
+ LOGGER.info(
+ "Pipe {}@{}: historical TsFile selection summary, selected by
progress uncovered {}, "
+ + "selected by unclosed/closing {}, filtered by time/path {}
(time {}, path {}), "
+ + "skipped covered {}, skipped deleted {}, skipped generated
by pipe {}, "
+ + "pin failed {}",
+ pipeName,
+ dataRegionId,
+ statistics.selectedByProgressUncoveredCount,
+ statistics.selectedByUnclosedOrClosingCount,
+ statistics.filteredByTimeOrPathCount,
+ statistics.filteredByTimeCount,
+ statistics.filteredByPathCount,
+ statistics.skippedCoveredCount,
+ statistics.skippedDeletedCount,
+ statistics.skippedGeneratedByPipeCount,
+ statistics.pinFailedCount);
} finally {
tsFileManager.readUnlock();
}
@@ -576,19 +595,22 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
return resource1.getTsFilePath().compareTo(resource2.getTsFilePath());
}
- private boolean shouldExtractTsFileResource(final TsFileResource resource) {
+ private boolean shouldExtractTsFileResource(
+ final TsFileResource resource, final
HistoricalTsFileExtractionStatistics statistics) {
if (!isHistoricalSourceEnabled) {
return false;
}
// Some resource is marked as deleted but not removed from the list.
if (resource.isDeleted()) {
+ ++statistics.skippedDeletedCount;
return false;
}
// Some resource is generated by pipe. We ignore them if the pipe should
not transfer pipe
// requests.
if (resource.isGeneratedByPipe() && !isForwardingPipeRequests) {
+ ++statistics.skippedGeneratedByPipeCount;
return false;
}
@@ -598,21 +620,28 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
&& Optional.ofNullable(resource.getProcessor())
.map(TsFileProcessor::alreadyMarkedClosing)
.orElse(true)) {
+ ++statistics.selectedByUnclosedOrClosingCount;
return true;
}
if (!mayTsFileContainUnprocessedData(resource)) {
+ ++statistics.skippedCoveredCount;
return false;
}
if (!isTsFileResourceOverlappedWithTimeRange(resource)) {
+ ++statistics.filteredByTimeOrPathCount;
+ ++statistics.filteredByTimeCount;
return false;
}
if (!mayTsFileResourceOverlappedWithPattern(resource)) {
+ ++statistics.filteredByTimeOrPathCount;
+ ++statistics.filteredByPathCount;
return false;
}
+ ++statistics.selectedByProgressUncoveredCount;
return true;
}
@@ -687,6 +716,19 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
:
progressIndex.getProgressIndexByType(TimeWindowStateProgressIndex.class).orElse(null);
}
+ private static class HistoricalTsFileExtractionStatistics {
+
+ private int selectedByProgressUncoveredCount;
+ private int selectedByUnclosedOrClosingCount;
+ private int filteredByTimeOrPathCount;
+ private int filteredByTimeCount;
+ private int filteredByPathCount;
+ private int skippedCoveredCount;
+ private int skippedDeletedCount;
+ private int skippedGeneratedByPipeCount;
+ private int pinFailedCount;
+ }
+
private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource
resource) {
// Trimming to avoid unnecessary file device getter
if (isDbNameCoveredByPattern) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
index 68b9fafdb92..17a0d63b221 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
@@ -154,7 +154,7 @@ public class DataNodeShutdownHook extends Thread {
final long shutdownProgressPersistTimeoutInMs =
PipeDataNodeAgent.task().getShutdownProgressPersistTimeoutInMs();
logger.info(
- "Persisting pipe progress indexes before shutdown, timeout {} ms.",
+ "Persisting pipe progress indexes before shutdown with timeout {} ms.",
shutdownProgressPersistTimeoutInMs);
if
(!PipeDataNodeAgent.task().persistAllProgressIndex(shutdownProgressPersistTimeoutInMs))
{
logger.warn("Pipe progress indexes were not confirmed during shutdown.");
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
index 29942d75104..3b264f749a2 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
@@ -48,7 +48,6 @@ import
org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeException;
-import com.google.common.collect.ImmutableMap;
import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.junit.Assert;
import org.junit.Test;
@@ -352,21 +351,41 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
tempDir,
"superset.tsfile",
hybridProgressIndex(
- new IoTProgressIndex(ImmutableMap.of(1, 100L, 2, 200L)),
+ iotProgressIndex(1, 100L, 2, 200L),
new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10))),
hybridProgressIndex(
- new IoTProgressIndex(1, 100L),
+ iotProgressIndex(1, 100L),
new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 9))),
false);
assertMayTsFileContainUnprocessedData(
tempDir,
"missing-dimension.tsfile",
- hybridProgressIndex(new IoTProgressIndex(1, 100L)),
+ hybridProgressIndex(iotProgressIndex(1, 100L)),
hybridProgressIndex(
- new IoTProgressIndex(1, 90L),
+ iotProgressIndex(1, 90L),
new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10))),
true);
+
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "larger-iot.tsfile",
+ hybridProgressIndex(
+ iotProgressIndex(1, 100L, 2, 200L),
+ new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10))),
+ hybridProgressIndex(
+ iotProgressIndex(1, 101L),
+ new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10))),
+ true);
+
+ final ProgressIndex recoverProgressIndex =
+ new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10));
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "old-sequence-recover.tsfile",
+ hybridProgressIndex(recoverProgressIndex, iotProgressIndex(1, 100L)),
+ recoverProgressIndex,
+ false);
} finally {
FileUtils.deleteFileOrDirectory(tempDir);
}
@@ -817,6 +836,16 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
return result;
}
+ private static IoTProgressIndex iotProgressIndex(
+ final int firstPeerId, final long firstSearchIndex, final long...
peerIdAndSearchIndexPairs) {
+ final Map<Integer, Long> peerId2SearchIndex = new HashMap<>();
+ peerId2SearchIndex.put(firstPeerId, firstSearchIndex);
+ for (int i = 0; i < peerIdAndSearchIndexPairs.length; i += 2) {
+ peerId2SearchIndex.put((int) peerIdAndSearchIndexPairs[i],
peerIdAndSearchIndexPairs[i + 1]);
+ }
+ return new IoTProgressIndex(peerId2SearchIndex);
+ }
+
private static void sortExtractedResources(
final PipeHistoricalDataRegionTsFileSource source, final
List<TsFileResource> resources)
throws ReflectiveOperationException {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeperTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeperTest.java
index bfd45b708b6..d7dcc30beac 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeperTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeperTest.java
@@ -35,6 +35,8 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
public class PipeTsFileEpochProgressIndexKeeperTest {
@@ -97,8 +99,7 @@ public class PipeTsFileEpochProgressIndexKeeperTest {
public void testProgressIndexCheckUsesEqualOrAfterCoverage() throws
IOException {
final ProgressIndex registeredProgressIndex =
hybridProgressIndex(
- new IoTProgressIndex(1, 90L),
- new RecoverProgressIndex(-1, new SimpleProgressIndex(0, 10)));
+ iotProgressIndex(1, 90L), new RecoverProgressIndex(-1, new
SimpleProgressIndex(0, 10)));
keeper.registerProgressIndex(
DATA_REGION_ID,
TASK_SCOPE_A,
@@ -106,7 +107,7 @@ public class PipeTsFileEpochProgressIndexKeeperTest {
Assert.assertFalse(
keeper.isProgressIndexAfterOrEquals(
- DATA_REGION_ID, TASK_SCOPE_A, "current.tsfile", new
IoTProgressIndex(1, 100L)));
+ DATA_REGION_ID, TASK_SCOPE_A, "current.tsfile",
iotProgressIndex(1, 100L)));
Assert.assertTrue(
keeper.isProgressIndexAfterOrEquals(
@@ -114,7 +115,7 @@ public class PipeTsFileEpochProgressIndexKeeperTest {
TASK_SCOPE_A,
"current.tsfile",
hybridProgressIndex(
- new IoTProgressIndex(1, 100L),
+ iotProgressIndex(1, 100L),
new RecoverProgressIndex(-1, new SimpleProgressIndex(0,
10)))));
}
@@ -157,4 +158,10 @@ public class PipeTsFileEpochProgressIndexKeeperTest {
}
return result;
}
+
+ private IoTProgressIndex iotProgressIndex(final int peerId, final long
searchIndex) {
+ final Map<Integer, Long> peerId2SearchIndex = new HashMap<>();
+ peerId2SearchIndex.put(peerId, searchIndex);
+ return new IoTProgressIndex(peerId2SearchIndex);
+ }
}