This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new c9e36dd  [7264] Add Exception to Broker Response When Not All Segments 
Are Available (Partial Response) (#7397)
c9e36dd is described below

commit c9e36dd0a53370d7b8dc24459426dc77497eb8ac
Author: suddendust <[email protected]>
AuthorDate: Sat Oct 2 11:37:57 2021 +0530

    [7264] Add Exception to Broker Response When Not All Segments Are Available 
(Partial Response) (#7397)
    
    Add exception to the broker response in the following cases:
    1. numUnavailableSegments > 0. Implying there are some unavailable segments.
    2. numSegmentsQueried > numSegmentsAcquired. That is, the servers could not 
acquire some segments.
    3. numServersQueried > numServersResponded. That is, some servers did not 
respond.
---
 .../requesthandler/BaseBrokerRequestHandler.java   |  25 ++-
 .../SingleConnectionBrokerRequestHandler.java      |  13 +-
 .../pinot/common/exception/QueryException.java     |   6 +
 .../core/data/manager/BaseTableDataManager.java    |   4 +-
 .../core/data/manager/InstanceDataManager.java     |   5 +
 .../query/executor/ServerQueryExecutorV1Impl.java  |  64 ++++---
 .../BaseTableDataManagerAcquireSegmentTest.java    |   2 +-
 .../executor/QueryExecutorExceptionsTest.java      | 188 +++++++++++++++++++++
 .../tests/OfflineClusterIntegrationTest.java       |  12 +-
 .../local/data/manager/TableDataManager.java       |   3 +-
 .../starter/helix/HelixInstanceDataManager.java    |   5 +
 11 files changed, 277 insertions(+), 50 deletions(-)

diff --git 
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
 
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 61c6d88..b767fc0 100644
--- 
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++ 
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -70,6 +70,7 @@ import org.apache.pinot.common.request.SelectionSort;
 import org.apache.pinot.common.request.transform.TransformExpressionTree;
 import org.apache.pinot.common.response.BrokerResponse;
 import org.apache.pinot.common.response.broker.BrokerResponseNative;
+import org.apache.pinot.common.response.broker.QueryProcessingException;
 import org.apache.pinot.common.response.broker.ResultTable;
 import org.apache.pinot.common.utils.DataSchema;
 import org.apache.pinot.common.utils.helix.TableCache;
@@ -407,12 +408,12 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
     long routingStartTimeNs = System.nanoTime();
     Map<ServerInstance, List<String>> offlineRoutingTable = null;
     Map<ServerInstance, List<String>> realtimeRoutingTable = null;
-    int numUnavailableSegments = 0;
+    List<String> unavailableSegments = new ArrayList<>();
     if (offlineBrokerRequest != null) {
       // NOTE: Routing table might be null if table is just removed
       RoutingTable routingTable = 
_routingManager.getRoutingTable(offlineBrokerRequest);
       if (routingTable != null) {
-        numUnavailableSegments += routingTable.getUnavailableSegments().size();
+        unavailableSegments.addAll(routingTable.getUnavailableSegments());
         Map<ServerInstance, List<String>> serverInstanceToSegmentsMap = 
routingTable.getServerInstanceToSegmentsMap();
         if (!serverInstanceToSegmentsMap.isEmpty()) {
           offlineRoutingTable = serverInstanceToSegmentsMap;
@@ -427,7 +428,7 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
       // NOTE: Routing table might be null if table is just removed
       RoutingTable routingTable = 
_routingManager.getRoutingTable(realtimeBrokerRequest);
       if (routingTable != null) {
-        numUnavailableSegments += routingTable.getUnavailableSegments().size();
+        unavailableSegments.addAll(routingTable.getUnavailableSegments());
         Map<ServerInstance, List<String>> serverInstanceToSegmentsMap = 
routingTable.getServerInstanceToSegmentsMap();
         if (!serverInstanceToSegmentsMap.isEmpty()) {
           realtimeRoutingTable = serverInstanceToSegmentsMap;
@@ -438,6 +439,7 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
         realtimeBrokerRequest = null;
       }
     }
+    int numUnavailableSegments = unavailableSegments.size();
     requestStatistics.setNumUnavailableSegments(numUnavailableSegments);
 
     if (offlineBrokerRequest == null && realtimeBrokerRequest == null) {
@@ -487,6 +489,11 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
       _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, 1);
     }
 
+    if (numUnavailableSegments != 0) {
+      brokerResponse.addToExceptions(new 
QueryProcessingException(QueryException.BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE,
+          String.format("%d segments %s unavailable", numUnavailableSegments, 
unavailableSegments)));
+    }
+
     // Set total query processing time
     long totalTimeMs = TimeUnit.NANOSECONDS.toMillis(executionEndTimeNs - 
compilationStartTimeNs);
     brokerResponse.setTimeUsedMs(totalTimeMs);
@@ -715,12 +722,12 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
     long routingStartTimeNs = System.nanoTime();
     Map<ServerInstance, List<String>> offlineRoutingTable = null;
     Map<ServerInstance, List<String>> realtimeRoutingTable = null;
-    int numUnavailableSegments = 0;
+    List<String> unavailableSegments = new ArrayList<>();
     if (offlineBrokerRequest != null) {
       // NOTE: Routing table might be null if table is just removed
       RoutingTable routingTable = 
_routingManager.getRoutingTable(offlineBrokerRequest);
       if (routingTable != null) {
-        numUnavailableSegments += routingTable.getUnavailableSegments().size();
+        unavailableSegments.addAll(routingTable.getUnavailableSegments());
         Map<ServerInstance, List<String>> serverInstanceToSegmentsMap = 
routingTable.getServerInstanceToSegmentsMap();
         if (!serverInstanceToSegmentsMap.isEmpty()) {
           offlineRoutingTable = serverInstanceToSegmentsMap;
@@ -735,7 +742,7 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
       // NOTE: Routing table might be null if table is just removed
       RoutingTable routingTable = 
_routingManager.getRoutingTable(realtimeBrokerRequest);
       if (routingTable != null) {
-        numUnavailableSegments += routingTable.getUnavailableSegments().size();
+        unavailableSegments.addAll(routingTable.getUnavailableSegments());
         Map<ServerInstance, List<String>> serverInstanceToSegmentsMap = 
routingTable.getServerInstanceToSegmentsMap();
         if (!serverInstanceToSegmentsMap.isEmpty()) {
           realtimeRoutingTable = serverInstanceToSegmentsMap;
@@ -746,6 +753,7 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
         realtimeBrokerRequest = null;
       }
     }
+    int numUnavailableSegments = unavailableSegments.size();
     requestStatistics.setNumUnavailableSegments(numUnavailableSegments);
 
     if (offlineBrokerRequest == null && realtimeBrokerRequest == null) {
@@ -794,6 +802,11 @@ public abstract class BaseBrokerRequestHandler implements 
BrokerRequestHandler {
       _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, 1);
     }
 
+    if (numUnavailableSegments != 0) {
+      brokerResponse.addToExceptions(new 
QueryProcessingException(QueryException.BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE,
+          String.format("%d segments %s unavailable", numUnavailableSegments, 
unavailableSegments)));
+    }
+
     // Set total query processing time
     long totalTimeMs = TimeUnit.NANOSECONDS.toMillis(executionEndTimeNs - 
compilationStartTimeNs);
     brokerResponse.setTimeUsedMs(totalTimeMs);
diff --git 
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
 
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
index 0abf690..76deaa5 100644
--- 
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
+++ 
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.broker.requesthandler;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -95,12 +96,15 @@ public class SingleConnectionBrokerRequestHandler extends 
BaseBrokerRequestHandl
     int numServersQueried = response.size();
     long totalResponseSize = 0;
     Map<ServerRoutingInstance, DataTable> dataTableMap = new 
HashMap<>(HashUtil.getHashMapCapacity(numServersQueried));
+    List<ServerRoutingInstance> serversNotResponded = new ArrayList<>();
     for (Map.Entry<ServerRoutingInstance, ServerResponse> entry : 
response.entrySet()) {
       ServerResponse serverResponse = entry.getValue();
       DataTable dataTable = serverResponse.getDataTable();
       if (dataTable != null) {
         dataTableMap.put(entry.getKey(), dataTable);
         totalResponseSize += serverResponse.getResponseSize();
+      } else {
+        serversNotResponded.add(entry.getKey());
       }
     }
     int numServersResponded = dataTableMap.size();
@@ -122,12 +126,15 @@ public class SingleConnectionBrokerRequestHandler extends 
BaseBrokerRequestHandl
       brokerResponse
           .addToExceptions(new 
QueryProcessingException(QueryException.BROKER_REQUEST_SEND_ERROR_CODE, 
errorMsg));
     }
+    int numServersNotResponded = serversNotResponded.size();
+    if (numServersNotResponded != 0) {
+      brokerResponse.addToExceptions(new 
QueryProcessingException(QueryException.SERVER_NOT_RESPONDING_ERROR_CODE,
+          String.format("%d servers %s not responded", numServersNotResponded, 
serversNotResponded)));
+      _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
+    }
     if (brokerResponse.getExceptionsSize() > 0) {
       _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1);
     }
-    if (numServersQueried > numServersResponded) {
-      _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
-    }
     _brokerMetrics.addMeteredTableValue(rawTableName, 
BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, totalResponseSize);
 
     return brokerResponse;
diff --git 
a/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
 
b/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
index 755960f..0cacd51 100644
--- 
a/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
+++ 
b/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
@@ -57,15 +57,18 @@ public class QueryException {
   public static final int SERVER_SHUTTING_DOWN_ERROR_CODE = 210;
   public static final int SERVER_OUT_OF_CAPACITY_ERROR_CODE = 211;
   public static final int SERVER_TABLE_MISSING_ERROR_CODE = 230;
+  public static final int SERVER_SEGMENT_MISSING_ERROR_CODE = 235;
   public static final int QUERY_SCHEDULING_TIMEOUT_ERROR_CODE = 240;
   public static final int EXECUTION_TIMEOUT_ERROR_CODE = 250;
   public static final int BROKER_GATHER_ERROR_CODE = 300;
+  public static final int BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE = 305;
   public static final int DATA_TABLE_DESERIALIZATION_ERROR_CODE = 310;
   public static final int FUTURE_CALL_ERROR_CODE = 350;
   public static final int BROKER_TIMEOUT_ERROR_CODE = 400;
   public static final int BROKER_RESOURCE_MISSING_ERROR_CODE = 410;
   public static final int BROKER_INSTANCE_MISSING_ERROR_CODE = 420;
   public static final int BROKER_REQUEST_SEND_ERROR_CODE = 425;
+  public static final int SERVER_NOT_RESPONDING_ERROR_CODE = 427;
   public static final int TOO_MANY_REQUESTS_ERROR_CODE = 429;
   public static final int INTERNAL_ERROR_CODE = 450;
   public static final int MERGE_RESPONSE_ERROR_CODE = 500;
@@ -92,6 +95,8 @@ public class QueryException {
       new ProcessingException(SERVER_OUT_OF_CAPACITY_ERROR_CODE);
   public static final ProcessingException SERVER_TABLE_MISSING_ERROR =
       new ProcessingException(SERVER_TABLE_MISSING_ERROR_CODE);
+  public static final ProcessingException SERVER_SEGMENT_MISSING_ERROR =
+      new ProcessingException(SERVER_SEGMENT_MISSING_ERROR_CODE);
   public static final ProcessingException QUERY_SCHEDULING_TIMEOUT_ERROR =
       new ProcessingException(QUERY_SCHEDULING_TIMEOUT_ERROR_CODE);
   public static final ProcessingException EXECUTION_TIMEOUT_ERROR =
@@ -126,6 +131,7 @@ public class QueryException {
     SERVER_SCHEDULER_DOWN_ERROR.setMessage("ServerShuttingDown");
     SERVER_OUT_OF_CAPACITY_ERROR.setMessage("ServerOutOfCapacity");
     SERVER_TABLE_MISSING_ERROR.setMessage("ServerTableMissing");
+    SERVER_SEGMENT_MISSING_ERROR.setMessage("ServerSegmentMissing");
     QUERY_SCHEDULING_TIMEOUT_ERROR.setMessage("QuerySchedulingTimeoutError");
     EXECUTION_TIMEOUT_ERROR.setMessage("ExecutionTimeoutError");
     BROKER_GATHER_ERROR.setMessage("BrokerGatherError");
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index 99bb521..43f4d9c 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -200,12 +200,14 @@ public abstract class BaseTableDataManager implements 
TableDataManager {
   }
 
   @Override
-  public List<SegmentDataManager> acquireSegments(List<String> segmentNames) {
+  public List<SegmentDataManager> acquireSegments(List<String> segmentNames, 
List<String> missingSegments) {
     List<SegmentDataManager> segmentDataManagers = new ArrayList<>();
     for (String segmentName : segmentNames) {
       SegmentDataManager segmentDataManager = 
_segmentDataManagerMap.get(segmentName);
       if (segmentDataManager != null && 
segmentDataManager.increaseReferenceCount()) {
         segmentDataManagers.add(segmentDataManager);
+      } else {
+        missingSegments.add(segmentName);
       }
     }
     return segmentDataManagers;
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
index 66074a8..94546c0 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
@@ -49,6 +49,11 @@ public interface InstanceDataManager {
       throws ConfigurationException;
 
   /**
+   * Returns the instance id.
+   */
+  String getInstanceId();
+
+  /**
    * Starts the data manager.
    * <p>Should be called only once after data manager gets initialized but 
before calling any other method.
    */
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
index bd97762..28e3db8 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
@@ -155,34 +155,21 @@ public class ServerQueryExecutorV1Impl implements 
QueryExecutor {
     }
 
     List<String> segmentsToQuery = queryRequest.getSegmentsToQuery();
-    List<SegmentDataManager> segmentDataManagers = 
tableDataManager.acquireSegments(segmentsToQuery);
-
-    // When segment is removed from the IdealState:
-    // 1. Controller schedules a state transition to server to turn segment 
OFFLINE
-    // 2. Server gets the state transition, removes the segment data manager 
and update its CurrentState
-    // 3. Controller gathers the CurrentState and update the ExternalView
-    // 4. Broker watches ExternalView change and updates the routing table to 
stop querying the segment
-    //
-    // After step 2 but before step 4, segment will be missing on server side
-    // TODO: Change broker to watch both IdealState and ExternalView to not 
query the removed segments
-    int numSegmentsQueried = segmentsToQuery.size();
+    List<String> missingSegments = new ArrayList<>();
+    List<SegmentDataManager> segmentDataManagers = 
tableDataManager.acquireSegments(segmentsToQuery, missingSegments);
     int numSegmentsAcquired = segmentDataManagers.size();
-    if (numSegmentsQueried > numSegmentsAcquired) {
-      _serverMetrics.addMeteredTableValue(tableNameWithType, 
ServerMeter.NUM_MISSING_SEGMENTS,
-          numSegmentsQueried - numSegmentsAcquired);
-    }
     List<IndexSegment> indexSegments = new ArrayList<>(numSegmentsAcquired);
     for (SegmentDataManager segmentDataManager : segmentDataManagers) {
       indexSegments.add(segmentDataManager.getSegment());
     }
 
     // Gather stats for realtime consuming segments
-    int numConsumingSegmentsProcessed = 0;
+    int numConsumingSegments = 0;
     long minIndexTimeMs = Long.MAX_VALUE;
     long minIngestionTimeMs = Long.MAX_VALUE;
     for (IndexSegment indexSegment : indexSegments) {
       if (indexSegment instanceof MutableSegment) {
-        numConsumingSegmentsProcessed += 1;
+        numConsumingSegments += 1;
         SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
         long indexTimeMs = segmentMetadata.getLastIndexedTimestamp();
         if (indexTimeMs != Long.MIN_VALUE && indexTimeMs < minIndexTimeMs) {
@@ -195,17 +182,6 @@ public class ServerQueryExecutorV1Impl implements 
QueryExecutor {
       }
     }
 
-    long minConsumingFreshnessTimeMs = minIngestionTimeMs;
-    if (numConsumingSegmentsProcessed > 0) {
-      if (minIngestionTimeMs == Long.MAX_VALUE) {
-        LOGGER.debug("Did not find valid ingestionTimestamp across consuming 
segments! Using indexTime instead");
-        minConsumingFreshnessTimeMs = minIndexTimeMs;
-      }
-      LOGGER
-          .debug("Querying: {} consuming segments with 
minConsumingFreshnessTimeMs: {}", numConsumingSegmentsProcessed,
-              minConsumingFreshnessTimeMs);
-    }
-
     boolean enableTrace = queryRequest.isEnableTrace();
     if (enableTrace) {
       TraceContext.register(requestId);
@@ -241,14 +217,32 @@ public class ServerQueryExecutorV1Impl implements 
QueryExecutor {
 
     queryProcessingTimer.stopAndRecord();
     long queryProcessingTime = queryProcessingTimer.getDurationMs();
-    dataTable.getMetadata().put(MetadataKey.NUM_SEGMENTS_QUERIED.getName(), 
Integer.toString(numSegmentsQueried));
-    dataTable.getMetadata().put(MetadataKey.TIME_USED_MS.getName(), 
Long.toString(queryProcessingTime));
+    Map<String, String> metadata = dataTable.getMetadata();
+    metadata.put(MetadataKey.NUM_SEGMENTS_QUERIED.getName(), 
Integer.toString(numSegmentsAcquired));
+    metadata.put(MetadataKey.TIME_USED_MS.getName(), 
Long.toString(queryProcessingTime));
+
+    // When segment is removed from the IdealState:
+    // 1. Controller schedules a state transition to server to turn segment 
OFFLINE
+    // 2. Server gets the state transition, removes the segment data manager 
and update its CurrentState
+    // 3. Controller gathers the CurrentState and update the ExternalView
+    // 4. Broker watches ExternalView change and updates the routing table to 
stop querying the segment
+    //
+    // After step 2 but before step 4, segment will be missing on server side
+    // TODO: Change broker to watch both IdealState and ExternalView to not 
query the removed segments
+    int numMissingSegments = missingSegments.size();
+    if (numMissingSegments != 0) {
+      
dataTable.addException(QueryException.getException(QueryException.SERVER_SEGMENT_MISSING_ERROR,
+          String.format("%d segments %s missing on server: %s", 
numMissingSegments, missingSegments,
+              _instanceDataManager.getInstanceId())));
+      _serverMetrics.addMeteredTableValue(tableNameWithType, 
ServerMeter.NUM_MISSING_SEGMENTS, numMissingSegments);
+    }
 
-    if (numConsumingSegmentsProcessed > 0) {
-      dataTable.getMetadata()
-          .put(MetadataKey.NUM_CONSUMING_SEGMENTS_PROCESSED.getName(), 
Integer.toString(numConsumingSegmentsProcessed));
-      dataTable.getMetadata()
-          .put(MetadataKey.MIN_CONSUMING_FRESHNESS_TIME_MS.getName(), 
Long.toString(minConsumingFreshnessTimeMs));
+    if (numConsumingSegments > 0) {
+      long minConsumingFreshnessTimeMs = minIngestionTimeMs != Long.MAX_VALUE 
? minIngestionTimeMs : minIndexTimeMs;
+      LOGGER.debug("Request {} queried {} consuming segments with 
minConsumingFreshnessTimeMs: {}", requestId,
+          numConsumingSegments, minConsumingFreshnessTimeMs);
+      metadata.put(MetadataKey.NUM_CONSUMING_SEGMENTS_PROCESSED.getName(), 
Integer.toString(numConsumingSegments));
+      metadata.put(MetadataKey.MIN_CONSUMING_FRESHNESS_TIME_MS.getName(), 
Long.toString(minConsumingFreshnessTimeMs));
     }
 
     LOGGER.debug("Query processing time for request Id - {}: {}", requestId, 
queryProcessingTime);
diff --git 
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
 
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
index a34b569..9c03d51 100644
--- 
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
+++ 
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
@@ -335,7 +335,7 @@ public class BaseTableDataManagerAcquireSegmentTest {
             for (Integer segmentId : segmentIds) {
               segmentList.add(SEGMENT_PREFIX + segmentId);
             }
-            segmentDataManagers = 
_tableDataManager.acquireSegments(segmentList);
+            segmentDataManagers = 
_tableDataManager.acquireSegments(segmentList, new ArrayList<>());
           }
           // Some of them may be rejected, but that is OK.
 
diff --git 
a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
 
b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
new file mode 100644
index 0000000..88fb116
--- /dev/null
+++ 
b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.core.query.executor;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.helix.HelixManager;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+import org.apache.pinot.common.exception.QueryException;
+import org.apache.pinot.common.metrics.PinotMetricUtils;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.request.InstanceRequest;
+import org.apache.pinot.common.utils.DataTable;
+import org.apache.pinot.core.data.manager.InstanceDataManager;
+import org.apache.pinot.core.data.manager.offline.TableDataManagerProvider;
+import org.apache.pinot.core.query.request.ServerQueryRequest;
+import org.apache.pinot.parsers.QueryCompiler;
+import org.apache.pinot.segment.local.data.manager.TableDataManager;
+import org.apache.pinot.segment.local.data.manager.TableDataManagerConfig;
+import 
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
+import 
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.IngestionSchemaValidator;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.FileFormat;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEqualsNoOrder;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class QueryExecutorExceptionsTest {
+  private static final String AVRO_DATA_PATH = "data/simpleData200001.avro";
+  private static final String EMPTY_JSON_DATA_PATH = 
"data/test_empty_data.json";
+  private static final String QUERY_EXECUTOR_CONFIG_PATH = 
"conf/query-executor.properties";
+  private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), 
"QueryExecutorTest");
+  private static final String TABLE_NAME = "testTable";
+  private static final int NUM_SEGMENTS_TO_GENERATE = 2;
+  private static final int NUM_EMPTY_SEGMENTS_TO_GENERATE = 2;
+  private static final QueryCompiler COMPILER = new CalciteSqlCompiler();
+  private static final ExecutorService QUERY_RUNNERS = 
Executors.newFixedThreadPool(20);
+
+  private final List<ImmutableSegment> _indexSegments = new 
ArrayList<>(NUM_SEGMENTS_TO_GENERATE);
+  private final List<String> _segmentNames = new 
ArrayList<>(NUM_SEGMENTS_TO_GENERATE);
+
+  private ServerMetrics _serverMetrics;
+  private QueryExecutor _queryExecutor;
+
+  @BeforeClass
+  public void setUp()
+      throws Exception {
+    // Set up the segments
+    FileUtils.deleteQuietly(INDEX_DIR);
+    assertTrue(INDEX_DIR.mkdirs());
+    URL resourceUrl = getClass().getClassLoader().getResource(AVRO_DATA_PATH);
+    assertNotNull(resourceUrl);
+    File avroFile = new File(resourceUrl.getFile());
+    Schema schema = 
SegmentTestUtils.extractSchemaFromAvroWithoutTime(avroFile);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
+    for (int i = 0; i < NUM_SEGMENTS_TO_GENERATE; i++) {
+      SegmentGeneratorConfig config =
+          SegmentTestUtils.getSegmentGeneratorConfig(avroFile, 
FileFormat.AVRO, INDEX_DIR, TABLE_NAME, tableConfig,
+              schema);
+      config.setSegmentNamePostfix(Integer.toString(i));
+      SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
+      driver.init(config);
+      driver.build();
+      IngestionSchemaValidator ingestionSchemaValidator = 
driver.getIngestionSchemaValidator();
+      
assertFalse(ingestionSchemaValidator.getDataTypeMismatchResult().isMismatchDetected());
+      
assertFalse(ingestionSchemaValidator.getSingleValueMultiValueFieldMismatchResult().isMismatchDetected());
+      
assertFalse(ingestionSchemaValidator.getMultiValueStructureMismatchResult().isMismatchDetected());
+      
assertFalse(ingestionSchemaValidator.getMissingPinotColumnResult().isMismatchDetected());
+      _indexSegments.add(ImmutableSegmentLoader.load(new File(INDEX_DIR, 
driver.getSegmentName()), ReadMode.mmap));
+      _segmentNames.add(driver.getSegmentName());
+    }
+    resourceUrl = 
getClass().getClassLoader().getResource(EMPTY_JSON_DATA_PATH);
+    assertNotNull(resourceUrl);
+    File jsonFile = new File(resourceUrl.getFile());
+    for (int i = NUM_SEGMENTS_TO_GENERATE; i < NUM_SEGMENTS_TO_GENERATE + 
NUM_EMPTY_SEGMENTS_TO_GENERATE; i++) {
+      SegmentGeneratorConfig config =
+          SegmentTestUtils.getSegmentGeneratorConfig(jsonFile, 
FileFormat.JSON, INDEX_DIR, TABLE_NAME, tableConfig,
+              schema);
+      config.setSegmentNamePostfix(Integer.toString(i));
+      SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
+      driver.init(config);
+      driver.build();
+      _indexSegments.add(ImmutableSegmentLoader.load(new File(INDEX_DIR, 
driver.getSegmentName()), ReadMode.mmap));
+      _segmentNames.add(driver.getSegmentName());
+    }
+
+    // Mock the instance data manager
+    _serverMetrics = new 
ServerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
+    TableDataManagerConfig tableDataManagerConfig = 
mock(TableDataManagerConfig.class);
+    
when(tableDataManagerConfig.getTableDataManagerType()).thenReturn("OFFLINE");
+    when(tableDataManagerConfig.getTableName()).thenReturn(TABLE_NAME);
+    
when(tableDataManagerConfig.getDataDir()).thenReturn(FileUtils.getTempDirectoryPath());
+    @SuppressWarnings("unchecked")
+    TableDataManager tableDataManager =
+        TableDataManagerProvider.getTableDataManager(tableDataManagerConfig, 
"testInstance",
+            mock(ZkHelixPropertyStore.class), mock(ServerMetrics.class), 
mock(HelixManager.class), null);
+    tableDataManager.start();
+    //we don't add index segments to the data manager to simulate 
numSegmentsAcquired < numSegmentsQueried
+    InstanceDataManager instanceDataManager = mock(InstanceDataManager.class);
+    
when(instanceDataManager.getTableDataManager(TABLE_NAME)).thenReturn(tableDataManager);
+
+    // Set up the query executor
+    resourceUrl = 
getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
+    assertNotNull(resourceUrl);
+    PropertiesConfiguration queryExecutorConfig = new 
PropertiesConfiguration();
+    queryExecutorConfig.setDelimiterParsingDisabled(false);
+    queryExecutorConfig.load(new File(resourceUrl.getFile()));
+    _queryExecutor = new ServerQueryExecutorV1Impl();
+    _queryExecutor.init(new PinotConfiguration(queryExecutorConfig), 
instanceDataManager, _serverMetrics);
+  }
+
+  /**
+   * Given some segments were missing, when a query is executed, then the 
correct error code is returned along with
+   * the list of missing segments.
+   */
+  @Test
+  public void testServerSegmentMissingExceptionDetails() {
+    String query = "SELECT COUNT(*) FROM " + TABLE_NAME;
+    InstanceRequest instanceRequest = new InstanceRequest(0L, 
COMPILER.compileToBrokerRequest(query));
+    instanceRequest.setSearchSegments(_segmentNames);
+    DataTable instanceResponse = 
_queryExecutor.processQuery(getQueryRequest(instanceRequest), QUERY_RUNNERS);
+    Map<Integer, String> exceptions = instanceResponse.getExceptions();
+    
assertTrue(exceptions.containsKey(QueryException.SERVER_SEGMENT_MISSING_ERROR_CODE));
+
+    String errorMessage = 
exceptions.get(QueryException.SERVER_SEGMENT_MISSING_ERROR_CODE);
+    String[] actualMissingSegments = StringUtils.splitByWholeSeparator(
+        errorMessage.substring(1 + errorMessage.indexOf('['), 
errorMessage.indexOf(']')), ", ");
+    String[] expectedMissingSegments = new String[]{"testTable_0", 
"testTable_1", "testTable_2", "testTable_3"};
+    assertEqualsNoOrder(actualMissingSegments, expectedMissingSegments);
+  }
+
+  @AfterClass
+  public void tearDown() {
+    for (IndexSegment segment : _indexSegments) {
+      segment.destroy();
+    }
+    FileUtils.deleteQuietly(INDEX_DIR);
+  }
+
+  private ServerQueryRequest getQueryRequest(InstanceRequest instanceRequest) {
+    return new ServerQueryRequest(instanceRequest, _serverMetrics, 
System.currentTimeMillis());
+  }
+}
diff --git 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 7f3652a..cd707d2 100644
--- 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -267,16 +267,22 @@ public class OfflineClusterIntegrationTest extends 
BaseClusterIntegrationTestSet
       try {
         JsonNode queryResponse = postQuery(TEST_TIMEOUT_QUERY);
         JsonNode exceptions = queryResponse.get("exceptions");
-        if (!exceptions.isEmpty()) {
+        if (exceptions.isEmpty()) {
+          return false;
+        }
+        int errorCode = exceptions.get(0).get("errorCode").asInt();
+        if (errorCode == QueryException.BROKER_TIMEOUT_ERROR_CODE) {
           // Timed out on broker side
-          return exceptions.get(0).get("errorCode").asInt() == 
QueryException.BROKER_TIMEOUT_ERROR_CODE;
-        } else {
+          return true;
+        }
+        if (errorCode == QueryException.SERVER_NOT_RESPONDING_ERROR_CODE) {
           // Timed out on server side
           int numServersQueried = 
queryResponse.get("numServersQueried").asInt();
           int numServersResponded = 
queryResponse.get("numServersResponded").asInt();
           int numDocsScanned = queryResponse.get("numDocsScanned").asInt();
           return numServersQueried == getNumServers() && numServersResponded 
== 0 && numDocsScanned == 0;
         }
+        return false;
       } catch (Exception e) {
         throw new RuntimeException(e);
       }
diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
index 36634db..22853ed 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
@@ -120,9 +120,10 @@ public interface TableDataManager {
    * <p>It is the caller's responsibility to return the segments by calling 
{@link #releaseSegment(SegmentDataManager)}.
    *
    * @param segmentNames List of names of the segment to acquire
+   * @param missingSegments Holder for segments unable to be acquired
    * @return List of segment data managers
    */
-  List<SegmentDataManager> acquireSegments(List<String> segmentNames);
+  List<SegmentDataManager> acquireSegments(List<String> segmentNames, 
List<String> missingSegments);
 
   /**
    * Acquires the segments with the given segment name.
diff --git 
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
 
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index 3cf0bbb..e639200 100644
--- 
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++ 
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -122,6 +122,11 @@ public class HelixInstanceDataManager implements 
InstanceDataManager {
   }
 
   @Override
+  public String getInstanceId() {
+    return _instanceId;
+  }
+
+  @Override
   public synchronized void start() {
     _propertyStore = _helixManager.getHelixPropertyStore();
     LOGGER.info("Helix instance data manager started");

---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to