This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7de438e434 Add segment level debug API (#9609)
7de438e434 is described below
commit 7de438e434f84da094554fec8922bd7d8b23faac
Author: Saurabh Dubey <[email protected]>
AuthorDate: Thu Oct 20 23:14:58 2022 +0530
Add segment level debug API (#9609)
---
...{TableDebugResource.java => DebugResource.java} | 84 ++++++++++++++++++++--
.../pinot/server/api/resources/DebugResource.java | 34 ++++++++-
2 files changed, 112 insertions(+), 6 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
similarity index 83%
rename from
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java
rename to
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
index d10c29f1e6..badd427bcc 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Executor;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -89,9 +90,9 @@ import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Api(tags = Constants.CLUSTER_TAG, authorizations = {@Authorization(value =
SWAGGER_AUTHORIZATION_KEY)})
@SwaggerDefinition(securityDefinition =
@SecurityDefinition(apiKeyAuthDefinitions = @ApiKeyAuthDefinition(name =
HttpHeaders.AUTHORIZATION, in =
ApiKeyAuthDefinition.ApiKeyLocation.HEADER, key = SWAGGER_AUTHORIZATION_KEY)))
-@Path("/")
-public class TableDebugResource {
- private static final Logger LOGGER =
LoggerFactory.getLogger(TableDebugResource.class);
+@Path("/debug/")
+public class DebugResource {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(DebugResource.class);
@Inject
PinotHelixResourceManager _pinotHelixResourceManager;
@@ -112,7 +113,7 @@ public class TableDebugResource {
ControllerConf _controllerConf;
@GET
- @Path("/debug/tables/{tableName}")
+ @Path("tables/{tableName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get debug information for table.", notes = "Debug
information for table.")
@ApiResponses(value = {
@@ -142,6 +143,22 @@ public class TableDebugResource {
return JsonUtils.objectToPrettyString(tableDebugInfos);
}
+ @GET
+ @Path("segments/{tableName}/{segmentName}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get debug information for segment.", notes = "Debug
information for segment.")
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404,
message = "Segment not found"),
+ @ApiResponse(code = 500, message = "Internal server error")
+ })
+ public TableDebugInfo.SegmentDebugInfo getSegmentDebugInfo(
+ @ApiParam(value = "Name of the table (with type)", required = true)
@PathParam("tableName")
+ String tableNameWithType,
+ @ApiParam(value = "Name of the segment", required = true)
@PathParam("segmentName") String segmentName)
+ throws Exception {
+ return debugSegment(tableNameWithType, segmentName);
+ }
+
/**
* Helper method to collect debug information about the table.
*
@@ -217,6 +234,65 @@ public class TableDebugResource {
tableSizeDetails._estimatedSizeInBytes) : new
TableDebugInfo.TableSizeSummary(-1, -1);
}
+ private TableDebugInfo.SegmentDebugInfo debugSegment(String
tableNameWithType, String segmentName)
+ throws IOException {
+ IdealState idealState =
_pinotHelixResourceManager.getTableIdealState(tableNameWithType);
+ if (idealState == null) {
+ return null;
+ }
+
+ ExternalView externalView =
_pinotHelixResourceManager.getTableExternalView(tableNameWithType);
+ Map<String, String> evStateMap = (externalView != null) ?
externalView.getStateMap(segmentName) : null;
+
+ Map<String, String> isServerToStateMap =
idealState.getRecord().getMapFields().get(segmentName);
+ Set<String> serversHostingSegment =
_pinotHelixResourceManager.getServers(tableNameWithType, segmentName);
+
+ int serverRequestTimeoutMs =
_controllerConf.getServerAdminRequestTimeoutSeconds() * 1000;
+ BiMap<String, String> serverToEndpoints;
+ try {
+ serverToEndpoints =
_pinotHelixResourceManager.getDataInstanceAdminEndpoints(serversHostingSegment);
+ } catch (InvalidConfigException e) {
+ throw new WebApplicationException(
+ "Caught exception when getting segment debug info for table: " +
tableNameWithType);
+ }
+
+ List<String> serverUrls = new ArrayList<>(serverToEndpoints.size());
+ BiMap<String, String> endpointsToServers = serverToEndpoints.inverse();
+ for (String endpoint : endpointsToServers.keySet()) {
+ String segmentDebugInfoURI = String.format("%s/debug/segments/%s/%s",
endpoint, tableNameWithType, segmentName);
+ serverUrls.add(segmentDebugInfoURI);
+ }
+
+ CompletionServiceHelper completionServiceHelper =
+ new CompletionServiceHelper(_executor, _connectionManager,
endpointsToServers);
+ CompletionServiceHelper.CompletionServiceResponse serviceResponse =
+ completionServiceHelper.doMultiGetRequest(serverUrls,
tableNameWithType, false, serverRequestTimeoutMs);
+
+ Map<String, SegmentServerDebugInfo> serverToSegmentDebugInfo = new
HashMap<>();
+ for (Map.Entry<String, String> streamResponse :
serviceResponse._httpResponses.entrySet()) {
+ SegmentServerDebugInfo segmentDebugInfo =
+ JsonUtils.stringToObject(streamResponse.getValue(),
SegmentServerDebugInfo.class);
+ serverToSegmentDebugInfo.put(streamResponse.getKey(), segmentDebugInfo);
+ }
+
+ Map<String, TableDebugInfo.SegmentState> segmentServerState = new
HashMap<>();
+ for (String instanceName : isServerToStateMap.keySet()) {
+ String isState = isServerToStateMap.get(instanceName);
+ String evState = (evStateMap != null) ? evStateMap.get(instanceName) :
null;
+ SegmentServerDebugInfo segmentServerDebugInfo =
serverToSegmentDebugInfo.get(instanceName);
+
+ if (segmentServerDebugInfo != null) {
+ segmentServerState.put(instanceName,
+ new TableDebugInfo.SegmentState(isState, evState,
segmentServerDebugInfo.getSegmentSize(),
+ segmentServerDebugInfo.getConsumerInfo(),
segmentServerDebugInfo.getErrorInfo()));
+ } else {
+ segmentServerState.put(instanceName, new
TableDebugInfo.SegmentState(isState, evState, null, null, null));
+ }
+ }
+
+ return new TableDebugInfo.SegmentDebugInfo(segmentName,
segmentServerState);
+ }
+
/**
* Helper method to debug segments. Computes differences between ideal state
and external view for each segment.
*
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
index 9d0a1976fc..9132f28556 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
@@ -75,12 +75,42 @@ public class DebugResource {
@ApiOperation(value = "Get segments debug info for this table",
notes = "This is a debug endpoint, and won't maintain backward
compatibility")
public List<SegmentServerDebugInfo> getSegmentsDebugInfo(
- @ApiParam(value = "Name of the table", required = true)
@PathParam("tableName") String tableNameWithType) {
-
+ @ApiParam(value = "Name of the table (with type)", required = true)
@PathParam("tableName")
+ String tableNameWithType) {
TableType tableType =
TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
return getSegmentServerDebugInfo(tableNameWithType, tableType);
}
+ @GET
+ @Path("segments/{tableName}/{segmentName}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get segment debug info",
+ notes = "This is a debug endpoint, and won't maintain backward
compatibility")
+ public SegmentServerDebugInfo getSegmentDebugInfo(
+ @ApiParam(value = "Name of the table (with type)", required = true)
@PathParam("tableName")
+ String tableNameWithType,
+ @ApiParam(value = "Name of the segment", required = true)
@PathParam("segmentName") String segmentName) {
+ TableType tableType =
TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
+ TableDataManager tableDataManager =
+ ServerResourceUtils.checkGetTableDataManager(_serverInstance,
tableNameWithType);
+ Map<String, SegmentErrorInfo> segmentErrorsMap =
tableDataManager.getSegmentErrors();
+ SegmentDataManager segmentDataManager =
tableDataManager.acquireSegment(segmentName);
+ try {
+ SegmentConsumerInfo segmentConsumerInfo =
getSegmentConsumerInfo(segmentDataManager, tableType);
+ long segmentSize = getSegmentSize(segmentDataManager);
+ SegmentErrorInfo segmentErrorInfo = segmentErrorsMap.get(segmentName);
+ return new SegmentServerDebugInfo(segmentName,
FileUtils.byteCountToDisplaySize(segmentSize), segmentConsumerInfo,
+ segmentErrorInfo);
+ } catch (Exception e) {
+ throw new WebApplicationException(
+ "Caught exception when getting consumer info for table: " +
tableNameWithType + " segment: " + segmentName);
+ } finally {
+ if (segmentDataManager != null) {
+ tableDataManager.releaseSegment(segmentDataManager);
+ }
+ }
+ }
+
private List<SegmentServerDebugInfo> getSegmentServerDebugInfo(String
tableNameWithType, TableType tableType) {
List<SegmentServerDebugInfo> segmentServerDebugInfos = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
