This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c9e36dd [7264] Add Exception to Broker Response When Not All Segments
Are Available (Partial Response) (#7397)
c9e36dd is described below
commit c9e36dd0a53370d7b8dc24459426dc77497eb8ac
Author: suddendust <[email protected]>
AuthorDate: Sat Oct 2 11:37:57 2021 +0530
[7264] Add Exception to Broker Response When Not All Segments Are Available
(Partial Response) (#7397)
Add exception to the broker response in the following cases:
1. numUnavailableSegments > 0. Implying there are some unavailable segments.
2. numSegmentsQueried > numSegmentsAcquired. That is, the servers could not
acquire some segments.
3. numServersQueried > numServersResponded. That is, some servers did not
respond.
---
.../requesthandler/BaseBrokerRequestHandler.java | 25 ++-
.../SingleConnectionBrokerRequestHandler.java | 13 +-
.../pinot/common/exception/QueryException.java | 6 +
.../core/data/manager/BaseTableDataManager.java | 4 +-
.../core/data/manager/InstanceDataManager.java | 5 +
.../query/executor/ServerQueryExecutorV1Impl.java | 64 ++++---
.../BaseTableDataManagerAcquireSegmentTest.java | 2 +-
.../executor/QueryExecutorExceptionsTest.java | 188 +++++++++++++++++++++
.../tests/OfflineClusterIntegrationTest.java | 12 +-
.../local/data/manager/TableDataManager.java | 3 +-
.../starter/helix/HelixInstanceDataManager.java | 5 +
11 files changed, 277 insertions(+), 50 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 61c6d88..b767fc0 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -70,6 +70,7 @@ import org.apache.pinot.common.request.SelectionSort;
import org.apache.pinot.common.request.transform.TransformExpressionTree;
import org.apache.pinot.common.response.BrokerResponse;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
+import org.apache.pinot.common.response.broker.QueryProcessingException;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.helix.TableCache;
@@ -407,12 +408,12 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
long routingStartTimeNs = System.nanoTime();
Map<ServerInstance, List<String>> offlineRoutingTable = null;
Map<ServerInstance, List<String>> realtimeRoutingTable = null;
- int numUnavailableSegments = 0;
+ List<String> unavailableSegments = new ArrayList<>();
if (offlineBrokerRequest != null) {
// NOTE: Routing table might be null if table is just removed
RoutingTable routingTable =
_routingManager.getRoutingTable(offlineBrokerRequest);
if (routingTable != null) {
- numUnavailableSegments += routingTable.getUnavailableSegments().size();
+ unavailableSegments.addAll(routingTable.getUnavailableSegments());
Map<ServerInstance, List<String>> serverInstanceToSegmentsMap =
routingTable.getServerInstanceToSegmentsMap();
if (!serverInstanceToSegmentsMap.isEmpty()) {
offlineRoutingTable = serverInstanceToSegmentsMap;
@@ -427,7 +428,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
// NOTE: Routing table might be null if table is just removed
RoutingTable routingTable =
_routingManager.getRoutingTable(realtimeBrokerRequest);
if (routingTable != null) {
- numUnavailableSegments += routingTable.getUnavailableSegments().size();
+ unavailableSegments.addAll(routingTable.getUnavailableSegments());
Map<ServerInstance, List<String>> serverInstanceToSegmentsMap =
routingTable.getServerInstanceToSegmentsMap();
if (!serverInstanceToSegmentsMap.isEmpty()) {
realtimeRoutingTable = serverInstanceToSegmentsMap;
@@ -438,6 +439,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
realtimeBrokerRequest = null;
}
}
+ int numUnavailableSegments = unavailableSegments.size();
requestStatistics.setNumUnavailableSegments(numUnavailableSegments);
if (offlineBrokerRequest == null && realtimeBrokerRequest == null) {
@@ -487,6 +489,11 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
_brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, 1);
}
+ if (numUnavailableSegments != 0) {
+ brokerResponse.addToExceptions(new
QueryProcessingException(QueryException.BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE,
+ String.format("%d segments %s unavailable", numUnavailableSegments,
unavailableSegments)));
+ }
+
// Set total query processing time
long totalTimeMs = TimeUnit.NANOSECONDS.toMillis(executionEndTimeNs -
compilationStartTimeNs);
brokerResponse.setTimeUsedMs(totalTimeMs);
@@ -715,12 +722,12 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
long routingStartTimeNs = System.nanoTime();
Map<ServerInstance, List<String>> offlineRoutingTable = null;
Map<ServerInstance, List<String>> realtimeRoutingTable = null;
- int numUnavailableSegments = 0;
+ List<String> unavailableSegments = new ArrayList<>();
if (offlineBrokerRequest != null) {
// NOTE: Routing table might be null if table is just removed
RoutingTable routingTable =
_routingManager.getRoutingTable(offlineBrokerRequest);
if (routingTable != null) {
- numUnavailableSegments += routingTable.getUnavailableSegments().size();
+ unavailableSegments.addAll(routingTable.getUnavailableSegments());
Map<ServerInstance, List<String>> serverInstanceToSegmentsMap =
routingTable.getServerInstanceToSegmentsMap();
if (!serverInstanceToSegmentsMap.isEmpty()) {
offlineRoutingTable = serverInstanceToSegmentsMap;
@@ -735,7 +742,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
// NOTE: Routing table might be null if table is just removed
RoutingTable routingTable =
_routingManager.getRoutingTable(realtimeBrokerRequest);
if (routingTable != null) {
- numUnavailableSegments += routingTable.getUnavailableSegments().size();
+ unavailableSegments.addAll(routingTable.getUnavailableSegments());
Map<ServerInstance, List<String>> serverInstanceToSegmentsMap =
routingTable.getServerInstanceToSegmentsMap();
if (!serverInstanceToSegmentsMap.isEmpty()) {
realtimeRoutingTable = serverInstanceToSegmentsMap;
@@ -746,6 +753,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
realtimeBrokerRequest = null;
}
}
+ int numUnavailableSegments = unavailableSegments.size();
requestStatistics.setNumUnavailableSegments(numUnavailableSegments);
if (offlineBrokerRequest == null && realtimeBrokerRequest == null) {
@@ -794,6 +802,11 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
_brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, 1);
}
+ if (numUnavailableSegments != 0) {
+ brokerResponse.addToExceptions(new
QueryProcessingException(QueryException.BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE,
+ String.format("%d segments %s unavailable", numUnavailableSegments,
unavailableSegments)));
+ }
+
// Set total query processing time
long totalTimeMs = TimeUnit.NANOSECONDS.toMillis(executionEndTimeNs -
compilationStartTimeNs);
brokerResponse.setTimeUsedMs(totalTimeMs);
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
index 0abf690..76deaa5 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.broker.requesthandler;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,12 +96,15 @@ public class SingleConnectionBrokerRequestHandler extends
BaseBrokerRequestHandl
int numServersQueried = response.size();
long totalResponseSize = 0;
Map<ServerRoutingInstance, DataTable> dataTableMap = new
HashMap<>(HashUtil.getHashMapCapacity(numServersQueried));
+ List<ServerRoutingInstance> serversNotResponded = new ArrayList<>();
for (Map.Entry<ServerRoutingInstance, ServerResponse> entry :
response.entrySet()) {
ServerResponse serverResponse = entry.getValue();
DataTable dataTable = serverResponse.getDataTable();
if (dataTable != null) {
dataTableMap.put(entry.getKey(), dataTable);
totalResponseSize += serverResponse.getResponseSize();
+ } else {
+ serversNotResponded.add(entry.getKey());
}
}
int numServersResponded = dataTableMap.size();
@@ -122,12 +126,15 @@ public class SingleConnectionBrokerRequestHandler extends
BaseBrokerRequestHandl
brokerResponse
.addToExceptions(new
QueryProcessingException(QueryException.BROKER_REQUEST_SEND_ERROR_CODE,
errorMsg));
}
+ int numServersNotResponded = serversNotResponded.size();
+ if (numServersNotResponded != 0) {
+ brokerResponse.addToExceptions(new
QueryProcessingException(QueryException.SERVER_NOT_RESPONDING_ERROR_CODE,
+ String.format("%d servers %s not responded", numServersNotResponded,
serversNotResponded)));
+ _brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
+ }
if (brokerResponse.getExceptionsSize() > 0) {
_brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1);
}
- if (numServersQueried > numServersResponded) {
- _brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
- }
_brokerMetrics.addMeteredTableValue(rawTableName,
BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, totalResponseSize);
return brokerResponse;
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
b/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
index 755960f..0cacd51 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/exception/QueryException.java
@@ -57,15 +57,18 @@ public class QueryException {
public static final int SERVER_SHUTTING_DOWN_ERROR_CODE = 210;
public static final int SERVER_OUT_OF_CAPACITY_ERROR_CODE = 211;
public static final int SERVER_TABLE_MISSING_ERROR_CODE = 230;
+ public static final int SERVER_SEGMENT_MISSING_ERROR_CODE = 235;
public static final int QUERY_SCHEDULING_TIMEOUT_ERROR_CODE = 240;
public static final int EXECUTION_TIMEOUT_ERROR_CODE = 250;
public static final int BROKER_GATHER_ERROR_CODE = 300;
+ public static final int BROKER_SEGMENT_UNAVAILABLE_ERROR_CODE = 305;
public static final int DATA_TABLE_DESERIALIZATION_ERROR_CODE = 310;
public static final int FUTURE_CALL_ERROR_CODE = 350;
public static final int BROKER_TIMEOUT_ERROR_CODE = 400;
public static final int BROKER_RESOURCE_MISSING_ERROR_CODE = 410;
public static final int BROKER_INSTANCE_MISSING_ERROR_CODE = 420;
public static final int BROKER_REQUEST_SEND_ERROR_CODE = 425;
+ public static final int SERVER_NOT_RESPONDING_ERROR_CODE = 427;
public static final int TOO_MANY_REQUESTS_ERROR_CODE = 429;
public static final int INTERNAL_ERROR_CODE = 450;
public static final int MERGE_RESPONSE_ERROR_CODE = 500;
@@ -92,6 +95,8 @@ public class QueryException {
new ProcessingException(SERVER_OUT_OF_CAPACITY_ERROR_CODE);
public static final ProcessingException SERVER_TABLE_MISSING_ERROR =
new ProcessingException(SERVER_TABLE_MISSING_ERROR_CODE);
+ public static final ProcessingException SERVER_SEGMENT_MISSING_ERROR =
+ new ProcessingException(SERVER_SEGMENT_MISSING_ERROR_CODE);
public static final ProcessingException QUERY_SCHEDULING_TIMEOUT_ERROR =
new ProcessingException(QUERY_SCHEDULING_TIMEOUT_ERROR_CODE);
public static final ProcessingException EXECUTION_TIMEOUT_ERROR =
@@ -126,6 +131,7 @@ public class QueryException {
SERVER_SCHEDULER_DOWN_ERROR.setMessage("ServerShuttingDown");
SERVER_OUT_OF_CAPACITY_ERROR.setMessage("ServerOutOfCapacity");
SERVER_TABLE_MISSING_ERROR.setMessage("ServerTableMissing");
+ SERVER_SEGMENT_MISSING_ERROR.setMessage("ServerSegmentMissing");
QUERY_SCHEDULING_TIMEOUT_ERROR.setMessage("QuerySchedulingTimeoutError");
EXECUTION_TIMEOUT_ERROR.setMessage("ExecutionTimeoutError");
BROKER_GATHER_ERROR.setMessage("BrokerGatherError");
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index 99bb521..43f4d9c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -200,12 +200,14 @@ public abstract class BaseTableDataManager implements
TableDataManager {
}
@Override
- public List<SegmentDataManager> acquireSegments(List<String> segmentNames) {
+ public List<SegmentDataManager> acquireSegments(List<String> segmentNames,
List<String> missingSegments) {
List<SegmentDataManager> segmentDataManagers = new ArrayList<>();
for (String segmentName : segmentNames) {
SegmentDataManager segmentDataManager =
_segmentDataManagerMap.get(segmentName);
if (segmentDataManager != null &&
segmentDataManager.increaseReferenceCount()) {
segmentDataManagers.add(segmentDataManager);
+ } else {
+ missingSegments.add(segmentName);
}
}
return segmentDataManagers;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
index 66074a8..94546c0 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
@@ -49,6 +49,11 @@ public interface InstanceDataManager {
throws ConfigurationException;
/**
+ * Returns the instance id.
+ */
+ String getInstanceId();
+
+ /**
* Starts the data manager.
* <p>Should be called only once after data manager gets initialized but
before calling any other method.
*/
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
index bd97762..28e3db8 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/executor/ServerQueryExecutorV1Impl.java
@@ -155,34 +155,21 @@ public class ServerQueryExecutorV1Impl implements
QueryExecutor {
}
List<String> segmentsToQuery = queryRequest.getSegmentsToQuery();
- List<SegmentDataManager> segmentDataManagers =
tableDataManager.acquireSegments(segmentsToQuery);
-
- // When segment is removed from the IdealState:
- // 1. Controller schedules a state transition to server to turn segment
OFFLINE
- // 2. Server gets the state transition, removes the segment data manager
and update its CurrentState
- // 3. Controller gathers the CurrentState and update the ExternalView
- // 4. Broker watches ExternalView change and updates the routing table to
stop querying the segment
- //
- // After step 2 but before step 4, segment will be missing on server side
- // TODO: Change broker to watch both IdealState and ExternalView to not
query the removed segments
- int numSegmentsQueried = segmentsToQuery.size();
+ List<String> missingSegments = new ArrayList<>();
+ List<SegmentDataManager> segmentDataManagers =
tableDataManager.acquireSegments(segmentsToQuery, missingSegments);
int numSegmentsAcquired = segmentDataManagers.size();
- if (numSegmentsQueried > numSegmentsAcquired) {
- _serverMetrics.addMeteredTableValue(tableNameWithType,
ServerMeter.NUM_MISSING_SEGMENTS,
- numSegmentsQueried - numSegmentsAcquired);
- }
List<IndexSegment> indexSegments = new ArrayList<>(numSegmentsAcquired);
for (SegmentDataManager segmentDataManager : segmentDataManagers) {
indexSegments.add(segmentDataManager.getSegment());
}
// Gather stats for realtime consuming segments
- int numConsumingSegmentsProcessed = 0;
+ int numConsumingSegments = 0;
long minIndexTimeMs = Long.MAX_VALUE;
long minIngestionTimeMs = Long.MAX_VALUE;
for (IndexSegment indexSegment : indexSegments) {
if (indexSegment instanceof MutableSegment) {
- numConsumingSegmentsProcessed += 1;
+ numConsumingSegments += 1;
SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
long indexTimeMs = segmentMetadata.getLastIndexedTimestamp();
if (indexTimeMs != Long.MIN_VALUE && indexTimeMs < minIndexTimeMs) {
@@ -195,17 +182,6 @@ public class ServerQueryExecutorV1Impl implements
QueryExecutor {
}
}
- long minConsumingFreshnessTimeMs = minIngestionTimeMs;
- if (numConsumingSegmentsProcessed > 0) {
- if (minIngestionTimeMs == Long.MAX_VALUE) {
- LOGGER.debug("Did not find valid ingestionTimestamp across consuming
segments! Using indexTime instead");
- minConsumingFreshnessTimeMs = minIndexTimeMs;
- }
- LOGGER
- .debug("Querying: {} consuming segments with
minConsumingFreshnessTimeMs: {}", numConsumingSegmentsProcessed,
- minConsumingFreshnessTimeMs);
- }
-
boolean enableTrace = queryRequest.isEnableTrace();
if (enableTrace) {
TraceContext.register(requestId);
@@ -241,14 +217,32 @@ public class ServerQueryExecutorV1Impl implements
QueryExecutor {
queryProcessingTimer.stopAndRecord();
long queryProcessingTime = queryProcessingTimer.getDurationMs();
- dataTable.getMetadata().put(MetadataKey.NUM_SEGMENTS_QUERIED.getName(),
Integer.toString(numSegmentsQueried));
- dataTable.getMetadata().put(MetadataKey.TIME_USED_MS.getName(),
Long.toString(queryProcessingTime));
+ Map<String, String> metadata = dataTable.getMetadata();
+ metadata.put(MetadataKey.NUM_SEGMENTS_QUERIED.getName(),
Integer.toString(numSegmentsAcquired));
+ metadata.put(MetadataKey.TIME_USED_MS.getName(),
Long.toString(queryProcessingTime));
+
+ // When segment is removed from the IdealState:
+ // 1. Controller schedules a state transition to server to turn segment
OFFLINE
+ // 2. Server gets the state transition, removes the segment data manager
and update its CurrentState
+ // 3. Controller gathers the CurrentState and update the ExternalView
+ // 4. Broker watches ExternalView change and updates the routing table to
stop querying the segment
+ //
+ // After step 2 but before step 4, segment will be missing on server side
+ // TODO: Change broker to watch both IdealState and ExternalView to not
query the removed segments
+ int numMissingSegments = missingSegments.size();
+ if (numMissingSegments != 0) {
+
dataTable.addException(QueryException.getException(QueryException.SERVER_SEGMENT_MISSING_ERROR,
+ String.format("%d segments %s missing on server: %s",
numMissingSegments, missingSegments,
+ _instanceDataManager.getInstanceId())));
+ _serverMetrics.addMeteredTableValue(tableNameWithType,
ServerMeter.NUM_MISSING_SEGMENTS, numMissingSegments);
+ }
- if (numConsumingSegmentsProcessed > 0) {
- dataTable.getMetadata()
- .put(MetadataKey.NUM_CONSUMING_SEGMENTS_PROCESSED.getName(),
Integer.toString(numConsumingSegmentsProcessed));
- dataTable.getMetadata()
- .put(MetadataKey.MIN_CONSUMING_FRESHNESS_TIME_MS.getName(),
Long.toString(minConsumingFreshnessTimeMs));
+ if (numConsumingSegments > 0) {
+ long minConsumingFreshnessTimeMs = minIngestionTimeMs != Long.MAX_VALUE
? minIngestionTimeMs : minIndexTimeMs;
+ LOGGER.debug("Request {} queried {} consuming segments with
minConsumingFreshnessTimeMs: {}", requestId,
+ numConsumingSegments, minConsumingFreshnessTimeMs);
+ metadata.put(MetadataKey.NUM_CONSUMING_SEGMENTS_PROCESSED.getName(),
Integer.toString(numConsumingSegments));
+ metadata.put(MetadataKey.MIN_CONSUMING_FRESHNESS_TIME_MS.getName(),
Long.toString(minConsumingFreshnessTimeMs));
}
LOGGER.debug("Query processing time for request Id - {}: {}", requestId,
queryProcessingTime);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
index a34b569..9c03d51 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
@@ -335,7 +335,7 @@ public class BaseTableDataManagerAcquireSegmentTest {
for (Integer segmentId : segmentIds) {
segmentList.add(SEGMENT_PREFIX + segmentId);
}
- segmentDataManagers =
_tableDataManager.acquireSegments(segmentList);
+ segmentDataManagers =
_tableDataManager.acquireSegments(segmentList, new ArrayList<>());
}
// Some of them may be rejected, but that is OK.
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
new file mode 100644
index 0000000..88fb116
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.core.query.executor;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.helix.HelixManager;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+import org.apache.pinot.common.exception.QueryException;
+import org.apache.pinot.common.metrics.PinotMetricUtils;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.request.InstanceRequest;
+import org.apache.pinot.common.utils.DataTable;
+import org.apache.pinot.core.data.manager.InstanceDataManager;
+import org.apache.pinot.core.data.manager.offline.TableDataManagerProvider;
+import org.apache.pinot.core.query.request.ServerQueryRequest;
+import org.apache.pinot.parsers.QueryCompiler;
+import org.apache.pinot.segment.local.data.manager.TableDataManager;
+import org.apache.pinot.segment.local.data.manager.TableDataManagerConfig;
+import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.IngestionSchemaValidator;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.FileFormat;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEqualsNoOrder;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class QueryExecutorExceptionsTest {
+ private static final String AVRO_DATA_PATH = "data/simpleData200001.avro";
+ private static final String EMPTY_JSON_DATA_PATH =
"data/test_empty_data.json";
+ private static final String QUERY_EXECUTOR_CONFIG_PATH =
"conf/query-executor.properties";
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(),
"QueryExecutorTest");
+ private static final String TABLE_NAME = "testTable";
+ private static final int NUM_SEGMENTS_TO_GENERATE = 2;
+ private static final int NUM_EMPTY_SEGMENTS_TO_GENERATE = 2;
+ private static final QueryCompiler COMPILER = new CalciteSqlCompiler();
+ private static final ExecutorService QUERY_RUNNERS =
Executors.newFixedThreadPool(20);
+
+ private final List<ImmutableSegment> _indexSegments = new
ArrayList<>(NUM_SEGMENTS_TO_GENERATE);
+ private final List<String> _segmentNames = new
ArrayList<>(NUM_SEGMENTS_TO_GENERATE);
+
+ private ServerMetrics _serverMetrics;
+ private QueryExecutor _queryExecutor;
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ // Set up the segments
+ FileUtils.deleteQuietly(INDEX_DIR);
+ assertTrue(INDEX_DIR.mkdirs());
+ URL resourceUrl = getClass().getClassLoader().getResource(AVRO_DATA_PATH);
+ assertNotNull(resourceUrl);
+ File avroFile = new File(resourceUrl.getFile());
+ Schema schema =
SegmentTestUtils.extractSchemaFromAvroWithoutTime(avroFile);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();
+ for (int i = 0; i < NUM_SEGMENTS_TO_GENERATE; i++) {
+ SegmentGeneratorConfig config =
+ SegmentTestUtils.getSegmentGeneratorConfig(avroFile,
FileFormat.AVRO, INDEX_DIR, TABLE_NAME, tableConfig,
+ schema);
+ config.setSegmentNamePostfix(Integer.toString(i));
+ SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config);
+ driver.build();
+ IngestionSchemaValidator ingestionSchemaValidator =
driver.getIngestionSchemaValidator();
+
assertFalse(ingestionSchemaValidator.getDataTypeMismatchResult().isMismatchDetected());
+
assertFalse(ingestionSchemaValidator.getSingleValueMultiValueFieldMismatchResult().isMismatchDetected());
+
assertFalse(ingestionSchemaValidator.getMultiValueStructureMismatchResult().isMismatchDetected());
+
assertFalse(ingestionSchemaValidator.getMissingPinotColumnResult().isMismatchDetected());
+ _indexSegments.add(ImmutableSegmentLoader.load(new File(INDEX_DIR,
driver.getSegmentName()), ReadMode.mmap));
+ _segmentNames.add(driver.getSegmentName());
+ }
+ resourceUrl =
getClass().getClassLoader().getResource(EMPTY_JSON_DATA_PATH);
+ assertNotNull(resourceUrl);
+ File jsonFile = new File(resourceUrl.getFile());
+ for (int i = NUM_SEGMENTS_TO_GENERATE; i < NUM_SEGMENTS_TO_GENERATE +
NUM_EMPTY_SEGMENTS_TO_GENERATE; i++) {
+ SegmentGeneratorConfig config =
+ SegmentTestUtils.getSegmentGeneratorConfig(jsonFile,
FileFormat.JSON, INDEX_DIR, TABLE_NAME, tableConfig,
+ schema);
+ config.setSegmentNamePostfix(Integer.toString(i));
+ SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config);
+ driver.build();
+ _indexSegments.add(ImmutableSegmentLoader.load(new File(INDEX_DIR,
driver.getSegmentName()), ReadMode.mmap));
+ _segmentNames.add(driver.getSegmentName());
+ }
+
+ // Mock the instance data manager
+ _serverMetrics = new
ServerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
+ TableDataManagerConfig tableDataManagerConfig =
mock(TableDataManagerConfig.class);
+
when(tableDataManagerConfig.getTableDataManagerType()).thenReturn("OFFLINE");
+ when(tableDataManagerConfig.getTableName()).thenReturn(TABLE_NAME);
+
when(tableDataManagerConfig.getDataDir()).thenReturn(FileUtils.getTempDirectoryPath());
+ @SuppressWarnings("unchecked")
+ TableDataManager tableDataManager =
+ TableDataManagerProvider.getTableDataManager(tableDataManagerConfig,
"testInstance",
+ mock(ZkHelixPropertyStore.class), mock(ServerMetrics.class),
mock(HelixManager.class), null);
+ tableDataManager.start();
+ //we don't add index segments to the data manager to simulate
numSegmentsAcquired < numSegmentsQueried
+ InstanceDataManager instanceDataManager = mock(InstanceDataManager.class);
+
when(instanceDataManager.getTableDataManager(TABLE_NAME)).thenReturn(tableDataManager);
+
+ // Set up the query executor
+ resourceUrl =
getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
+ assertNotNull(resourceUrl);
+ PropertiesConfiguration queryExecutorConfig = new
PropertiesConfiguration();
+ queryExecutorConfig.setDelimiterParsingDisabled(false);
+ queryExecutorConfig.load(new File(resourceUrl.getFile()));
+ _queryExecutor = new ServerQueryExecutorV1Impl();
+ _queryExecutor.init(new PinotConfiguration(queryExecutorConfig),
instanceDataManager, _serverMetrics);
+ }
+
+ /**
+ * Given some segments were missing, when a query is executed, then the
correct error code is returned along with
+ * the list of missing segments.
+ */
+ @Test
+ public void testServerSegmentMissingExceptionDetails() {
+ String query = "SELECT COUNT(*) FROM " + TABLE_NAME;
+ InstanceRequest instanceRequest = new InstanceRequest(0L,
COMPILER.compileToBrokerRequest(query));
+ instanceRequest.setSearchSegments(_segmentNames);
+ DataTable instanceResponse =
_queryExecutor.processQuery(getQueryRequest(instanceRequest), QUERY_RUNNERS);
+ Map<Integer, String> exceptions = instanceResponse.getExceptions();
+
assertTrue(exceptions.containsKey(QueryException.SERVER_SEGMENT_MISSING_ERROR_CODE));
+
+ String errorMessage =
exceptions.get(QueryException.SERVER_SEGMENT_MISSING_ERROR_CODE);
+ String[] actualMissingSegments = StringUtils.splitByWholeSeparator(
+ errorMessage.substring(1 + errorMessage.indexOf('['),
errorMessage.indexOf(']')), ", ");
+ String[] expectedMissingSegments = new String[]{"testTable_0",
"testTable_1", "testTable_2", "testTable_3"};
+ assertEqualsNoOrder(actualMissingSegments, expectedMissingSegments);
+ }
+
+ @AfterClass
+ public void tearDown() {
+ for (IndexSegment segment : _indexSegments) {
+ segment.destroy();
+ }
+ FileUtils.deleteQuietly(INDEX_DIR);
+ }
+
+ private ServerQueryRequest getQueryRequest(InstanceRequest instanceRequest) {
+ return new ServerQueryRequest(instanceRequest, _serverMetrics,
System.currentTimeMillis());
+ }
+}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 7f3652a..cd707d2 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -267,16 +267,22 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
try {
JsonNode queryResponse = postQuery(TEST_TIMEOUT_QUERY);
JsonNode exceptions = queryResponse.get("exceptions");
- if (!exceptions.isEmpty()) {
+ if (exceptions.isEmpty()) {
+ return false;
+ }
+ int errorCode = exceptions.get(0).get("errorCode").asInt();
+ if (errorCode == QueryException.BROKER_TIMEOUT_ERROR_CODE) {
// Timed out on broker side
- return exceptions.get(0).get("errorCode").asInt() ==
QueryException.BROKER_TIMEOUT_ERROR_CODE;
- } else {
+ return true;
+ }
+ if (errorCode == QueryException.SERVER_NOT_RESPONDING_ERROR_CODE) {
// Timed out on server side
int numServersQueried =
queryResponse.get("numServersQueried").asInt();
int numServersResponded =
queryResponse.get("numServersResponded").asInt();
int numDocsScanned = queryResponse.get("numDocsScanned").asInt();
return numServersQueried == getNumServers() && numServersResponded
== 0 && numDocsScanned == 0;
}
+ return false;
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
index 36634db..22853ed 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManager.java
@@ -120,9 +120,10 @@ public interface TableDataManager {
* <p>It is the caller's responsibility to return the segments by calling
{@link #releaseSegment(SegmentDataManager)}.
*
* @param segmentNames List of names of the segment to acquire
+ * @param missingSegments Holder for segments unable to be acquired
* @return List of segment data managers
*/
- List<SegmentDataManager> acquireSegments(List<String> segmentNames);
+ List<SegmentDataManager> acquireSegments(List<String> segmentNames,
List<String> missingSegments);
/**
* Acquires the segments with the given segment name.
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index 3cf0bbb..e639200 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -122,6 +122,11 @@ public class HelixInstanceDataManager implements
InstanceDataManager {
}
@Override
+ public String getInstanceId() {
+ return _instanceId;
+ }
+
+ @Override
public synchronized void start() {
_propertyStore = _helixManager.getHelixPropertyStore();
LOGGER.info("Helix instance data manager started");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]