This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c2526941149 MSQ: Fix totalFiles tracking when querying data servers.
(#19104)
c2526941149 is described below
commit c25269411490cb35246993879257dddd1a408b63
Author: Gian Merlino <[email protected]>
AuthorDate: Mon Mar 9 12:52:39 2026 -0700
MSQ: Fix totalFiles tracking when querying data servers. (#19104)
When we query realtime servers, we gather up any handed-off segments
into a new SegmentsInputSlice and reattach it. This would cause
setTotalFiles to be called, which overwrites the previous value,
often with zero.
This patch changes setTotalFiles to addTotalFiles, meaning that
if any segments are handed-off and need to be fetched directly,
the total file count will go up.
---
.../src/main/java/org/apache/druid/msq/counters/ChannelCounters.java | 4 ++--
.../org/apache/druid/msq/input/external/ExternalInputSliceReader.java | 2 +-
.../org/apache/druid/msq/input/table/SegmentsInputSliceReader.java | 2 +-
.../java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/multi-stage-query/src/main/java/org/apache/druid/msq/counters/ChannelCounters.java
b/multi-stage-query/src/main/java/org/apache/druid/msq/counters/ChannelCounters.java
index fd2200512de..d7ae0a47924 100644
---
a/multi-stage-query/src/main/java/org/apache/druid/msq/counters/ChannelCounters.java
+++
b/multi-stage-query/src/main/java/org/apache/druid/msq/counters/ChannelCounters.java
@@ -123,11 +123,11 @@ public class ChannelCounters implements QueryCounter
add(partitionNumber, rac.numRows(), numBytes, 1, 0);
}
- public ChannelCounters setTotalFiles(final long nFiles)
+ public ChannelCounters addTotalFiles(final long nFiles)
{
synchronized (this) {
ensureCapacityForPartition(NO_PARTITION);
- totalFiles.set(NO_PARTITION, nFiles);
+ totalFiles.set(NO_PARTITION, totalFiles.getLong(NO_PARTITION) + nFiles);
return this;
}
}
diff --git
a/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalInputSliceReader.java
b/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalInputSliceReader.java
index 29349112095..310175b0ef8 100644
---
a/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalInputSliceReader.java
+++
b/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalInputSliceReader.java
@@ -88,7 +88,7 @@ public class ExternalInputSliceReader implements
InputSliceReader
{
final ExternalInputSlice externalInputSlice = (ExternalInputSlice) slice;
final ChannelCounters inputCounters =
counters.channel(CounterNames.inputChannel(inputNumber))
-
.setTotalFiles(slice.fileCount());
+
.addTotalFiles(slice.fileCount());
final List<LoadableSegment> loadableSegments = new ArrayList<>();
for (final InputSource inputSource : externalInputSlice.getInputSources())
{
diff --git
a/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/SegmentsInputSliceReader.java
b/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/SegmentsInputSliceReader.java
index 5a3d01dee11..7c46509e09e 100644
---
a/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/SegmentsInputSliceReader.java
+++
b/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/SegmentsInputSliceReader.java
@@ -69,7 +69,7 @@ public class SegmentsInputSliceReader implements
InputSliceReader
{
final SegmentsInputSlice segmentsInputSlice = (SegmentsInputSlice) slice;
final ChannelCounters inputCounters =
counters.channel(CounterNames.inputChannel(inputNumber))
-
.setTotalFiles(slice.fileCount());
+
.addTotalFiles(slice.fileCount());
final List<LoadableSegment> loadableSegments = new ArrayList<>();
final List<DataServerQueryHandler> queryableServers = new ArrayList<>();
diff --git
a/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
b/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
index 6084240765e..e314ade7280 100644
---
a/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
+++
b/multi-stage-query/src/test/java/org/apache/druid/msq/counters/CountersSnapshotTreeTest.java
@@ -44,7 +44,7 @@ public class CountersSnapshotTreeTest
final ChannelCounters channelCounters = new ChannelCounters();
channelCounters.addFile(10, 13);
- channelCounters.setTotalFiles(14);
+ channelCounters.addTotalFiles(14);
// fake load to set some counters
channelCounters.addLoad(new AcquireSegmentResult(null, 1234L, 1L, 1L));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
