xiangfu0 commented on code in PR #20237:
URL: https://github.com/apache/pinot/pull/20237#discussion_r4546931543
##########
pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java:
##########
@@ -131,38 +131,92 @@ private TableReloadJsonResponse
processSegmentMetadataReloadResponse(
public JsonNode getSegmentsMetadata(String tableNameWithType, @Nullable
List<String> columns,
@Nullable List<String> segments, int timeoutMs)
throws InvalidConfigException, IOException {
- Map<String, List<String>> serverToSegs =
- _pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType);
- BiMap<String, String> endpoints =
-
_pinotHelixResourceManager.getDataInstanceAdminEndpoints(serverToSegs.keySet());
- ServerSegmentMetadataReader reader =
- new ServerSegmentMetadataReader(_executor, _connectionManager,
- _pinotHelixResourceManager.getServerAdminAuthProvider());
-
- List<String> tableUrls = new ArrayList<>(serverToSegs.size());
- for (String server : serverToSegs.keySet()) {
- tableUrls.add(reader.generateTableMetadataServerURL(
- tableNameWithType, columns, segments, endpoints.get(server)));
- }
+ return getSegmentsMetadataInternal(tableNameWithType, columns, segments,
timeoutMs);
+ }
- CompletionServiceHelper cs = new CompletionServiceHelper(_executor,
_connectionManager, endpoints.inverse(),
+ /// Common helper used by both the new (server-level) and legacy
(segment-level) endpoints.
+ private JsonNode fetchAndAggregateMetadata(List<String> urls, BiMap<String,
String> endpoints, boolean perSegmentJson,
+ String tableNameWithType, int timeoutMs)
+ throws InvalidConfigException, IOException {
+ CompletionServiceHelper cs = new CompletionServiceHelper(_executor,
_connectionManager, endpoints,
_pinotHelixResourceManager.getServerAdminAuthProvider());
CompletionServiceHelper.CompletionServiceResponse resp =
- cs.doMultiGetRequest(tableUrls, tableNameWithType, false, timeoutMs);
+ cs.doMultiGetRequest(urls, tableNameWithType, perSegmentJson,
timeoutMs);
+ // all requests will fail if new server endpoint is not available
if (resp._failedResponseCount > 0) {
- throw new RuntimeException(String.format("Got %d failed responses from
total %d server instances",
- resp._failedResponseCount, tableUrls.size()));
+ throw new RuntimeException(String.format("Got %d failed responses from
total %d server instances. "
+ + "Falling back to legacy segment metadata api",
resp._failedResponseCount, urls.size()));
}
ObjectMapper mapper = new ObjectMapper();
ObjectNode aggregatedNode = mapper.createObjectNode();
for (String body : resp._httpResponses.values()) {
JsonNode node = JsonUtils.stringToJsonNode(body);
- node.properties().forEach(entry -> aggregatedNode.set(entry.getKey(),
entry.getValue()));
+ // legacy returns one JSON per segment; new returns one JSON with many
fields
+ if (perSegmentJson) {
+ String segmentName = node.get("segmentName").asText();
+ aggregatedNode.set(segmentName, node);
+ } else {
+ node.properties().forEach(entry -> aggregatedNode.set(entry.getKey(),
entry.getValue()));
+ }
}
return aggregatedNode;
}
+ private List<String> buildTableLevelUrls(Map<String, List<String>>
serverToSegs, BiMap<String, String> endpoints,
+ String tableNameWithType, List<String> columns, List<String>
segmentsFilter, ServerSegmentMetadataReader reader) {
+ List<String> urls = new ArrayList<>(serverToSegs.size());
+ for (String server : serverToSegs.keySet()) {
+ urls.add(reader.generateTableMetadataServerURL(
+ tableNameWithType, columns, segmentsFilter, endpoints.get(server)));
+ }
+ return urls;
+ }
+
+ private List<String> buildSegmentLevelUrls(Map<String, List<String>>
serverToSegs, BiMap<String, String> endpoints,
+ String tableNameWithType, List<String> columns, List<String>
segmentsFilter, ServerSegmentMetadataReader reader) {
+ List<String> urls = new ArrayList<>();
+ for (Map.Entry<String, List<String>> e : serverToSegs.entrySet()) {
+ for (String segment : e.getValue()) {
+ if (segmentsFilter == null || segmentsFilter.isEmpty()
+ || segmentsFilter.contains(segment)) {
+ urls.add(reader.generateSegmentMetadataServerURL(
+ tableNameWithType, segment, columns, endpoints.get(e.getKey())));
+ }
+ }
+ }
+ return urls;
+ }
+
+ private JsonNode getSegmentsMetadataInternal(String tableNameWithType,
@Nullable List<String> columns,
+ @Nullable List<String> segments, int timeoutMs)
+ throws InvalidConfigException, IOException {
+ Map<String, List<String>> serverToSegs =
+ _pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType);
+ BiMap<String, String> endpoints =
+
_pinotHelixResourceManager.getDataInstanceAdminEndpoints(serverToSegs.keySet());
+ ServerSegmentMetadataReader reader =
+ new ServerSegmentMetadataReader(_executor, _connectionManager,
+ _pinotHelixResourceManager.getServerAdminAuthProvider());
+
+ // try table level endpoint first
+ try {
+ List<String> tableUrls = buildTableLevelUrls(serverToSegs, endpoints,
+ tableNameWithType, columns, segments, reader);
+ return fetchAndAggregateMetadata(tableUrls, endpoints.inverse(),
/*perSegmentJson=*/false,
+ tableNameWithType, timeoutMs);
+ } catch (RuntimeException e) {
Review Comment:
This fallback is the reason for the revert: it absorbs table-level failures
such as a 414 on long `segments=` lists. But the restored tests only check that
it is *not* used. Could you add a case where the table-level endpoint returns
414 and the per-segment endpoint returns metadata? I checked this locally at
RF=2: all segments come back, with and without a filter.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
