This is an automated email from the ASF dual-hosted git repository.
swaminathanmanish pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3d148bafe29 Expose segment-level accessors on SegmentDataManager for
server observability endpoints (#18821)
3d148bafe29 is described below
commit 3d148bafe299fa118fcb9cc113faf30fbcb6a005
Author: Krishan Goyal <[email protected]>
AuthorDate: Thu Jun 25 11:01:46 2026 +0530
Expose segment-level accessors on SegmentDataManager for server
observability endpoints (#18821)
* Add getReportableSegments() to SegmentDataManager to make it more flexible
* Add getCrc() to SegmentDataManager to make it more flexible
* Add getTier() to SegmentDataManager to make it more flexible
* Use getReportableSegments in DebugResource
* Minor fixes
* Remove unused import in TableSizeResource
Fixes Checkstyle UnusedImports violation that was failing the pinot-server
build (and all downstream CI jobs).
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
---
.../local/data/manager/SegmentDataManager.java | 20 ++++
.../pinot/server/api/resources/DebugResource.java | 11 +-
.../server/api/resources/TableSizeResource.java | 16 ++-
.../server/api/resources/TableTierResource.java | 17 ++-
.../pinot/server/api/resources/TablesResource.java | 117 ++++++++++-----------
.../apache/pinot/server/api/BaseResourceTest.java | 18 +++-
6 files changed, 118 insertions(+), 81 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/SegmentDataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/SegmentDataManager.java
index d3e17be99e1..eb2026e63f8 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/SegmentDataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/SegmentDataManager.java
@@ -20,6 +20,8 @@ package org.apache.pinot.segment.local.data.manager;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
@@ -73,6 +75,12 @@ public abstract class SegmentDataManager {
public abstract IndexSegment getSegment();
+ @Nullable
+ public String getTier() {
+ IndexSegment segment = getSegment();
+ return segment instanceof ImmutableSegment ? ((ImmutableSegment)
segment).getTier() : null;
+ }
+
public boolean hasMultiSegments() {
return false;
}
@@ -81,6 +89,18 @@ public abstract class SegmentDataManager {
return List.of();
}
+ /**
+ * The index segment(s) this manager exposes.
+ * Defaults to the single backing segment.
+ */
+ public List<IndexSegment> getReportableSegments() {
+ return List.of(getSegment());
+ }
+
+ public String getCrc() {
+ return getSegment().getSegmentMetadata().getCrc();
+ }
+
/**
* Offloads the segment from the metadata management (e.g. upsert metadata),
but not releases the resources yet
* because there might be queries still accessing the segment.
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
index 3b7f2eb8ae9..f17e813ae41 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
@@ -48,13 +48,13 @@ import
org.apache.pinot.common.restlet.resources.SegmentConsumerInfo;
import org.apache.pinot.common.restlet.resources.SegmentErrorInfo;
import org.apache.pinot.common.restlet.resources.SegmentServerDebugInfo;
import org.apache.pinot.common.utils.DatabaseUtils;
-import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
import org.apache.pinot.core.data.manager.realtime.RealtimeSegmentDataManager;
import
org.apache.pinot.core.data.manager.realtime.RealtimeSegmentMetadataUtils;
import org.apache.pinot.core.data.manager.realtime.RealtimeTableDataManager;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.server.api.AdminApiApplication;
import org.apache.pinot.server.starter.ServerInstance;
import org.apache.pinot.spi.accounting.QueryResourceTracker;
@@ -203,8 +203,13 @@ public class DebugResource {
}
private long getSegmentSize(SegmentDataManager segmentDataManager) {
- return (segmentDataManager instanceof ImmutableSegmentDataManager) ?
((ImmutableSegment) segmentDataManager
- .getSegment()).getSegmentSizeBytes() : 0;
+ long size = 0;
+ for (IndexSegment segment : segmentDataManager.getReportableSegments()) {
+ if (segment instanceof ImmutableSegment) {
+ size += ((ImmutableSegment) segment).getSegmentSizeBytes();
+ }
+ }
+ return size;
}
private SegmentConsumerInfo getSegmentConsumerInfo(TableDataManager
tableDataManager,
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableSizeResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableSizeResource.java
index 489d4a6867e..88f88e95319 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableSizeResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableSizeResource.java
@@ -46,10 +46,10 @@ import
org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
import org.apache.pinot.common.restlet.resources.TableSizeInfo;
import org.apache.pinot.common.utils.DatabaseUtils;
import org.apache.pinot.core.data.manager.InstanceDataManager;
-import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.server.starter.ServerInstance;
import static org.apache.pinot.spi.utils.CommonConstants.DATABASE;
@@ -105,11 +105,17 @@ public class TableSizeResource {
List<SegmentDataManager> segmentDataManagers =
tableDataManager.acquireAllSegments();
try {
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
- if (segmentDataManager instanceof ImmutableSegmentDataManager) {
- ImmutableSegment immutableSegment = (ImmutableSegment)
segmentDataManager.getSegment();
- long segmentSizeBytes = immutableSegment.getSegmentSizeBytes();
+ long segmentSizeBytes = 0L;
+ boolean hasImmutable = false;
+ for (IndexSegment segment :
segmentDataManager.getReportableSegments()) {
+ if (segment instanceof ImmutableSegment immutableSegment) {
+ segmentSizeBytes += immutableSegment.getSegmentSizeBytes();
+ hasImmutable = true;
+ }
+ }
+ if (hasImmutable) {
if (detailed) {
- segmentSizeInfos.add(new
SegmentSizeInfo(immutableSegment.getSegmentName(), segmentSizeBytes));
+ segmentSizeInfos.add(new
SegmentSizeInfo(segmentDataManager.getSegmentName(), segmentSizeBytes));
}
tableSizeInBytes += segmentSizeBytes;
}
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableTierResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableTierResource.java
index b5b673cf6a5..542b1f55526 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableTierResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableTierResource.java
@@ -48,10 +48,9 @@ import
org.apache.pinot.common.restlet.resources.TableTierInfo;
import org.apache.pinot.common.utils.DatabaseUtils;
import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.core.data.manager.InstanceDataManager;
-import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
+import org.apache.pinot.core.data.manager.realtime.RealtimeSegmentDataManager;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
-import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.server.starter.ServerInstance;
import static org.apache.pinot.spi.utils.CommonConstants.DATABASE;
@@ -102,11 +101,10 @@ public class TableTierResource {
List<SegmentDataManager> segmentDataManagers =
tableDataManager.acquireAllSegments();
try {
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
- if (segmentDataManager instanceof ImmutableSegmentDataManager) {
- ImmutableSegment immutableSegment = (ImmutableSegment)
segmentDataManager.getSegment();
- segmentTiers.put(immutableSegment.getSegmentName(),
immutableSegment.getTier());
- } else {
+ if (segmentDataManager instanceof RealtimeSegmentDataManager) {
mutableSegments.add(segmentDataManager.getSegmentName());
+ } else {
+ segmentTiers.put(segmentDataManager.getSegmentName(),
segmentDataManager.getTier());
}
}
} finally {
@@ -153,11 +151,10 @@ public class TableTierResource {
Set<String> mutableSegments = new HashSet<>();
Map<String, String> segmentTiers = new HashMap<>();
try {
- if (segmentDataManager instanceof ImmutableSegmentDataManager) {
- ImmutableSegment immutableSegment = (ImmutableSegment)
segmentDataManager.getSegment();
- segmentTiers.put(immutableSegment.getSegmentName(),
immutableSegment.getTier());
- } else {
+ if (segmentDataManager instanceof RealtimeSegmentDataManager) {
mutableSegments.add(segmentDataManager.getSegmentName());
+ } else {
+ segmentTiers.put(segmentDataManager.getSegmentName(),
segmentDataManager.getTier());
}
} finally {
tableDataManager.releaseSegment(segmentDataManager);
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index dd3409d7703..013e4a3588a 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -231,54 +231,54 @@ public class TablesResource {
Map<String, Map<String, Double>> columnIndexSizesMap = new HashMap<>();
try {
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
- if (segmentDataManager instanceof ImmutableSegmentDataManager) {
- ImmutableSegment immutableSegment = (ImmutableSegment)
segmentDataManager.getSegment();
- long segmentSizeBytes = immutableSegment.getSegmentSizeBytes();
- SegmentMetadataImpl segmentMetadata =
- (SegmentMetadataImpl)
segmentDataManager.getSegment().getSegmentMetadata();
-
- totalSegmentSizeBytes += segmentSizeBytes;
- totalNumRows += segmentMetadata.getTotalDocs();
-
- if (columnSet == null) {
- columnSet = segmentMetadata.getAllColumns();
- } else {
- columnSet.retainAll(segmentMetadata.getAllColumns());
- }
- for (String column : columnSet) {
- ColumnMetadata columnMetadata =
segmentMetadata.getColumnMetadataMap().get(column);
- int columnLength = columnMetadata.getLengthOfLongestElement();
- if (columnLength < 0) {
- // For raw STRING/BYTES/BIG_DECIMAL column, set the columnLength
as the length of the max value.
- Comparable<?> maxValue = columnMetadata.getMaxValue();
- if (maxValue instanceof String) {
- columnLength = Utf8.encodedLength((String) maxValue);
- } else if (maxValue instanceof ByteArray) {
- columnLength = ((ByteArray) maxValue).length();
- } else if (maxValue instanceof BigDecimal) {
- columnLength = BigDecimalUtils.byteSize((BigDecimal) maxValue);
- } else {
- // For type of STRUCT, MAP, LIST, set the columnLength as
DEFAULT_MAX_LENGTH (512).
- columnLength = FieldSpec.DEFAULT_MAX_LENGTH;
- }
- }
- int columnCardinality = columnMetadata.getCardinality();
- columnLengthMap.merge(column, (double) columnLength, Double::sum);
- columnCardinalityMap.merge(column, (double) columnCardinality,
Double::sum);
- if (!columnMetadata.isSingleValue()) {
- int maxNumMultiValues =
columnMetadata.getMaxNumberOfMultiValues();
- maxNumMultiValuesMap.merge(column, (double) maxNumMultiValues,
Double::sum);
+ for (IndexSegment indexSegment :
segmentDataManager.getReportableSegments()) {
+ if (indexSegment instanceof ImmutableSegment immutableSegment) {
+ long segmentSizeBytes = immutableSegment.getSegmentSizeBytes();
+ SegmentMetadataImpl segmentMetadata = (SegmentMetadataImpl)
immutableSegment.getSegmentMetadata();
+
+ totalSegmentSizeBytes += segmentSizeBytes;
+ totalNumRows += segmentMetadata.getTotalDocs();
+
+ if (columnSet == null) {
+ columnSet = segmentMetadata.getAllColumns();
+ } else {
+ columnSet.retainAll(segmentMetadata.getAllColumns());
}
+ for (String column : columnSet) {
+ ColumnMetadata columnMetadata =
segmentMetadata.getColumnMetadataMap().get(column);
+ int columnLength = columnMetadata.getLengthOfLongestElement();
+ if (columnLength < 0) {
+ // For raw STRING/BYTES/BIG_DECIMAL column, set the
columnLength as the length of the max value.
+ Comparable<?> maxValue = columnMetadata.getMaxValue();
+ if (maxValue instanceof String) {
+ columnLength = Utf8.encodedLength((String) maxValue);
+ } else if (maxValue instanceof ByteArray) {
+ columnLength = ((ByteArray) maxValue).length();
+ } else if (maxValue instanceof BigDecimal) {
+ columnLength = BigDecimalUtils.byteSize((BigDecimal)
maxValue);
+ } else {
+ // For type of STRUCT, MAP, LIST, set the columnLength as
DEFAULT_MAX_LENGTH (512).
+ columnLength = FieldSpec.DEFAULT_MAX_LENGTH;
+ }
+ }
+ int columnCardinality = columnMetadata.getCardinality();
+ columnLengthMap.merge(column, (double) columnLength,
Double::sum);
+ columnCardinalityMap.merge(column, (double) columnCardinality,
Double::sum);
+ if (!columnMetadata.isSingleValue()) {
+ int maxNumMultiValues =
columnMetadata.getMaxNumberOfMultiValues();
+ maxNumMultiValuesMap.merge(column, (double) maxNumMultiValues,
Double::sum);
+ }
- IndexService indexService = IndexService.getInstance();
- for (int i = 0, n = columnMetadata.getNumIndexes(); i < n; i++) {
- String indexName =
indexService.get(columnMetadata.getIndexType(i)).getId();
- long value = columnMetadata.getIndexSize(i);
+ IndexService indexService = IndexService.getInstance();
+ for (int i = 0, n = columnMetadata.getNumIndexes(); i < n; i++) {
+ String indexName =
indexService.get(columnMetadata.getIndexType(i)).getId();
+ long value = columnMetadata.getIndexSize(i);
- Map<String, Double> columnIndexSizes =
columnIndexSizesMap.getOrDefault(column, new HashMap<>());
- Double indexSize = columnIndexSizes.getOrDefault(indexName, 0d)
+ value;
- columnIndexSizes.put(indexName, indexSize);
- columnIndexSizesMap.put(column, columnIndexSizes);
+ Map<String, Double> columnIndexSizes =
columnIndexSizesMap.getOrDefault(column, new HashMap<>());
+ Double indexSize = columnIndexSizes.getOrDefault(indexName,
0d) + value;
+ columnIndexSizes.put(indexName, indexSize);
+ columnIndexSizesMap.put(column, columnIndexSizes);
+ }
}
}
}
@@ -332,16 +332,17 @@ public class TablesResource {
// REALTIME segments may not have indexes since not all indexes have
mutable implementations
continue;
}
- totalSegmentCount++;
- IndexSegment segment = segmentDataManager.getSegment();
- segment.getColumnNames().forEach(col -> {
- columnToIndexesCount.putIfAbsent(col, new HashMap<>());
- DataSource colDataSource = segment.getDataSource(col);
- IndexService.getInstance().getAllIndexes().forEach(idxType -> {
- int count = colDataSource.getIndex(idxType) != null ? 1 : 0;
- columnToIndexesCount.get(col).merge(idxType.getId(), count,
Integer::sum);
+ for (IndexSegment segment :
segmentDataManager.getReportableSegments()) {
+ totalSegmentCount++;
+ segment.getColumnNames().forEach(col -> {
+ columnToIndexesCount.putIfAbsent(col, new HashMap<>());
+ DataSource colDataSource = segment.getDataSource(col);
+ IndexService.getInstance().getAllIndexes().forEach(idxType -> {
+ int count = colDataSource.getIndex(idxType) != null ? 1 : 0;
+ columnToIndexesCount.get(col).merge(idxType.getId(), count,
Integer::sum);
+ });
});
- });
+ }
}
TableIndexMetadataResponse tableIndexMetadataResponse =
new TableIndexMetadataResponse(totalSegmentCount,
columnToIndexesCount);
@@ -453,8 +454,7 @@ public class TablesResource {
try {
Map<String, String> segmentCrcForTable = new HashMap<>();
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
- segmentCrcForTable.put(segmentDataManager.getSegmentName(),
- segmentDataManager.getSegment().getSegmentMetadata().getCrc());
+ segmentCrcForTable.put(segmentDataManager.getSegmentName(),
segmentDataManager.getCrc());
}
return ResourceUtils.convertToJsonString(segmentCrcForTable);
} catch (Exception e) {
@@ -1181,12 +1181,11 @@ public class TablesResource {
String invalidReason = String.format(
"Segment %s is in ONLINE state, but segmentDataManager is
null", segmentName);
return new TableSegmentValidationInfo(false, invalidReason,
-1);
- } else if
(!segmentDataManager.getSegment().getSegmentMetadata().getCrc()
- .equals(String.valueOf(zkMetadata.getCrc()))) {
+ } else if
(!segmentDataManager.getCrc().equals(String.valueOf(zkMetadata.getCrc()))) {
String invalidReason = String.format(
"Segment %s is in ONLINE state, but has CRC mismatch. "
+ "zk_metadata_crc=%s, segment_data_manager_crc=%s",
- segmentName, zkMetadata.getCrc(),
segmentDataManager.getSegment().getSegmentMetadata().getCrc());
+ segmentName, zkMetadata.getCrc(),
segmentDataManager.getCrc());
return new TableSegmentValidationInfo(false, invalidReason,
-1);
}
maxEndTimeMs = Math.max(maxEndTimeMs, zkMetadata.getEndTimeMs());
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java
index 01993fb45c4..213dfb8608e 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/BaseResourceTest.java
@@ -19,8 +19,8 @@
package org.apache.pinot.server.api;
import java.io.File;
+import java.io.InputStream;
import java.net.URI;
-import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -107,9 +107,15 @@ public abstract class BaseResourceTest {
FileUtils.deleteQuietly(TEMP_DIR);
assertTrue(TEMP_DIR.mkdirs());
- URL resourceUrl =
getClass().getClassLoader().getResource(getAvroFileName());
- assertNotNull(resourceUrl);
- _avroFile = new File(resourceUrl.getFile());
+ // Copy the Avro fixture out of the classpath into TEMP_DIR so it is
always backed by a real file.
+ // The fixture may be served from a packaged test-jar when this base class
is reused from another
+ // module, in which case it cannot be opened as a plain File via the
resource URL.
+ String avroFileName = getAvroFileName();
+ _avroFile = new File(TEMP_DIR, new File(avroFileName).getName());
+ try (InputStream avroStream =
getClass().getClassLoader().getResourceAsStream(avroFileName)) {
+ assertNotNull(avroStream);
+ FileUtils.copyInputStreamToFile(avroStream, _avroFile);
+ }
// Mock the instance data manager
InstanceDataManager instanceDataManager = mock(InstanceDataManager.class);
@@ -154,6 +160,7 @@ public abstract class BaseResourceTest {
CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT);
_instanceId = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + hostname +
"_" + port;
serverConf.setProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_ID,
_instanceId);
+ configureServerConf(serverConf);
_adminApiApplication = new AdminApiApplication(_serverInstance, new
AllowAllAccessFactory(),
mock(ServerReloadJobStatusCache.class),
serverConf);
@@ -165,6 +172,9 @@ public abstract class BaseResourceTest {
String.format("http://%s:%d";, NetUtils.getHostAddress(),
CommonConstants.Server.DEFAULT_ADMIN_API_PORT));
}
+ protected void configureServerConf(PinotConfiguration serverConf) {
+ }
+
@AfterClass
public void tearDown() {
_adminApiApplication.stop();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
