This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e12bda1cf6 Fix the race condition of reading time boundary info (#8685)
e12bda1cf6 is described below
commit e12bda1cf6b9d0163a17ee545c0ae4cc00296612
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed May 11 18:08:11 2022 -0700
Fix the race condition of reading time boundary info (#8685)
There are 2 race conditions on the time boundary handling in the broker
request:
- We read time boundary info twice, once for the offline request and once
for the realtime request. If it gets updated between these 2 reads, we might
attach filter of different value to the offline and realtime request, and get
wrong result (less than expected).
- The time boundary info is available when the first segment is pushed to
the offline table. If we cannot find the time boundary, currently we don't
attach the time filter. If the segment becomes online after attaching the time
filter but before reading the routing table, we will query both offline and
realtime table without the time filter and get wrong result (more than expected)
Fix:
- Read time boundary info once, and use it to attach time filter for both
offline and realtime request
- If time boundary info is not available, do not query the offline side
---
.../requesthandler/BaseBrokerRequestHandler.java | 55 +++++++---------------
1 file changed, 18 insertions(+), 37 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 64d7f0db2b..5cf17eb19e 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -381,16 +381,29 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
// Prepare OFFLINE and REALTIME requests
BrokerRequest offlineBrokerRequest = null;
BrokerRequest realtimeBrokerRequest = null;
+ TimeBoundaryInfo timeBoundaryInfo = null;
Schema schema = _tableCache.getSchema(rawTableName);
+ if (offlineTableName != null && realtimeTableName != null) {
+ // Time boundary info might be null when there is no segment in the
offline table, query real-time side only
+ timeBoundaryInfo = _routingManager.getTimeBoundaryInfo(offlineTableName);
+ if (timeBoundaryInfo == null) {
+ LOGGER.debug("No time boundary info found for hybrid table: {}",
rawTableName);
+ offlineTableName = null;
+ }
+ }
if (offlineTableName != null && realtimeTableName != null) {
// Hybrid
- PinotQuery offlinePinotQuery = getOfflinePinotQuery(serverPinotQuery);
+ PinotQuery offlinePinotQuery = serverPinotQuery.deepCopy();
+ offlinePinotQuery.getDataSource().setTableName(offlineTableName);
+ attachTimeBoundary(offlinePinotQuery, timeBoundaryInfo, true);
handleExpressionOverride(offlinePinotQuery,
_tableCache.getExpressionOverrideMap(offlineTableName));
handleTimestampIndexOverride(offlinePinotQuery, offlineTableConfig);
_queryOptimizer.optimize(offlinePinotQuery, offlineTableConfig, schema);
offlineBrokerRequest =
CalciteSqlCompiler.convertToBrokerRequest(offlinePinotQuery);
- PinotQuery realtimePinotQuery = getRealtimePinotQuery(serverPinotQuery);
+ PinotQuery realtimePinotQuery = serverPinotQuery.deepCopy();
+ realtimePinotQuery.getDataSource().setTableName(realtimeTableName);
+ attachTimeBoundary(realtimePinotQuery, timeBoundaryInfo, false);
handleExpressionOverride(realtimePinotQuery,
_tableCache.getExpressionOverrideMap(realtimeTableName));
handleTimestampIndexOverride(realtimePinotQuery, realtimeTableConfig);
_queryOptimizer.optimize(realtimePinotQuery, realtimeTableConfig,
schema);
@@ -1563,42 +1576,10 @@ public abstract class BaseBrokerRequestHandler
implements BrokerRequestHandler {
}
/**
- * Helper method to create an OFFLINE query from the given hybrid query.
- * <p>This step will attach the time boundary to the request.
- */
- private PinotQuery getOfflinePinotQuery(PinotQuery hybridPinotQuery) {
- PinotQuery offlinePinotQuery = hybridPinotQuery.deepCopy();
- String rawTableName = hybridPinotQuery.getDataSource().getTableName();
- String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(rawTableName);
- offlinePinotQuery.getDataSource().setTableName(offlineTableName);
- attachTimeBoundary(rawTableName, offlinePinotQuery, true);
- return offlinePinotQuery;
- }
-
- /**
- * Helper method to create a REALTIME query from the given hybrid query.
- * <p>This step will attach the time boundary to the request.
+ * Helper method to attach the time boundary to the given PinotQuery.
*/
- private PinotQuery getRealtimePinotQuery(PinotQuery hybridPinotQuery) {
- PinotQuery realtimePinotQuery = hybridPinotQuery.deepCopy();
- String rawTableName = hybridPinotQuery.getDataSource().getTableName();
- String realtimeTableName =
TableNameBuilder.REALTIME.tableNameWithType(rawTableName);
- realtimePinotQuery.getDataSource().setTableName(realtimeTableName);
- attachTimeBoundary(rawTableName, realtimePinotQuery, false);
- return realtimePinotQuery;
- }
-
- /**
- * Helper method to attach the time boundary to the given broker request.
- */
- private void attachTimeBoundary(String rawTableName, PinotQuery pinotQuery,
boolean isOfflineRequest) {
- TimeBoundaryInfo timeBoundaryInfo =
-
_routingManager.getTimeBoundaryInfo(TableNameBuilder.OFFLINE.tableNameWithType(rawTableName));
- if (timeBoundaryInfo == null) {
- LOGGER.warn("Failed to find time boundary info for hybrid table: {}",
rawTableName);
- return;
- }
-
+ private static void attachTimeBoundary(PinotQuery pinotQuery,
TimeBoundaryInfo timeBoundaryInfo,
+ boolean isOfflineRequest) {
String timeColumn = timeBoundaryInfo.getTimeColumn();
String timeValue = timeBoundaryInfo.getTimeValue();
Expression timeFilterExpression = RequestUtils.getFunctionExpression(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
