This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0abd8f87ab [UI] Add controller UI to use multi-stage engine (#9072)
0abd8f87ab is described below
commit 0abd8f87abdffe304121982597e329213688ab8c
Author: Rong Rong <[email protected]>
AuthorDate: Tue Jul 26 13:22:19 2022 -0700
[UI] Add controller UI to use multi-stage engine (#9072)
* adding controller endpoint to poke multi-stage engine
* adding controller API to accept query option for using multistage engine
* changing the controller UI to show new Engine
* remove pql query options
* make non-supported exception more clear
Co-authored-by: Rong Rong <[email protected]>
---
.../broker/api/resources/PinotClientRequest.java | 15 +---
.../requesthandler/BaseBrokerRequestHandler.java | 6 +-
.../BrokerRequestHandlerDelegate.java | 10 ++-
.../requesthandler/BrokerRequestOptionsTest.java | 20 ++---
.../api/resources/PinotQueryResource.java | 92 +++++++++++++++-------
.../src/main/resources/app/pages/Query.tsx | 44 ++++++++---
.../main/resources/app/utils/PinotMethodUtils.ts | 43 +---------
.../tests/MultiStageEngineIntegrationTest.java | 3 +-
.../pinot/tools/MultistageEngineQuickStart.java | 10 +--
9 files changed, 130 insertions(+), 113 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
index 9e8037f329..9bc1b466c0 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
@@ -100,8 +100,6 @@ public class PinotClientRequest {
try {
ObjectNode requestJson = JsonUtils.newObjectNode();
requestJson.put(Request.SQL, query);
- String queryOptions = constructSqlQueryOptions();
- requestJson.put(Request.QUERY_OPTIONS, queryOptions);
if (traceEnabled != null) {
requestJson.put(Request.TRACE, traceEnabled);
}
@@ -133,10 +131,8 @@ public class PinotClientRequest {
if (!requestJson.has(Request.SQL)) {
throw new IllegalStateException("Payload is missing the query string
field 'sql'");
}
- String queryOptions = constructSqlQueryOptions();
- // the only query options as of now are sql related. do not allow any
custom query options in sql endpoint
- ObjectNode sqlRequestJson = ((ObjectNode)
requestJson).put(Request.QUERY_OPTIONS, queryOptions);
- BrokerResponse brokerResponse = executeSqlQuery(sqlRequestJson,
makeHttpIdentity(requestContext), false);
+ BrokerResponse brokerResponse = executeSqlQuery((ObjectNode)
requestJson, makeHttpIdentity(requestContext),
+ false);
asyncResponse.resume(brokerResponse.toJsonString());
} catch (Exception e) {
LOGGER.error("Caught exception while processing POST request", e);
@@ -174,13 +170,6 @@ public class PinotClientRequest {
new UnsupportedOperationException("Unsupported SQL type - " +
sqlType)));
}
}
-
- // TODO: Remove the SQL query options after releasing 0.11.0
- private String constructSqlQueryOptions() {
- return Request.QueryOptionKey.GROUP_BY_MODE + "=" + Request.SQL + ";" +
Request.QueryOptionKey.RESPONSE_FORMAT + "="
- + Request.SQL;
- }
-
private static HttpRequesterIdentity
makeHttpIdentity(org.glassfish.grizzly.http.server.Request context) {
Multimap<String, String> headers = ArrayListMultimap.create();
context.getHeaderNames().forEach(key ->
context.getHeaders(key).forEach(value -> headers.put(key, value)));
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 8320f17681..3f2bd37322 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -1420,7 +1420,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
throw new BadQueryRequestException("Unknown columnName '" + columnName +
"' found in the query");
}
- private static Map<String, String> getOptionsFromJson(JsonNode request,
String optionsKey) {
+ public static Map<String, String> getOptionsFromJson(JsonNode request,
String optionsKey) {
return
Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('=')
.split(request.get(optionsKey).asText());
}
@@ -1477,6 +1477,10 @@ public abstract class BaseBrokerRequestHandler
implements BrokerRequestHandler {
if (!queryOptions.isEmpty()) {
LOGGER.debug("Query options are set to: {} for request {}: {}",
queryOptions, requestId, query);
}
+ // TODO: Remove the SQL query options after releasing 0.11.0
+ // The query engine will break if these 2 options are missing during
version upgrade.
+ queryOptions.put(Broker.Request.QueryOptionKey.GROUP_BY_MODE,
Broker.Request.SQL);
+ queryOptions.put(Broker.Request.QueryOptionKey.RESPONSE_FORMAT,
Broker.Request.SQL);
}
/**
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
index a39ad1d265..046e60c04f 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
@@ -19,6 +19,7 @@
package org.apache.pinot.broker.requesthandler;
import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Map;
import javax.annotation.Nullable;
import org.apache.pinot.broker.api.RequesterIdentity;
import org.apache.pinot.common.response.BrokerResponse;
@@ -74,9 +75,12 @@ public class BrokerRequestHandlerDelegate implements
BrokerRequestHandler {
RequestContext requestContext)
throws Exception {
if (_isMultiStageQueryEngineEnabled && _multiStageWorkerRequestHandler !=
null) {
- JsonNode node =
request.get(CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE);
- if (node != null && node.asBoolean()) {
- return _multiStageWorkerRequestHandler.handleRequest(request,
requesterIdentity, requestContext);
+ if (request.has("queryOptions")) {
+ Map<String, String> queryOptionMap =
BaseBrokerRequestHandler.getOptionsFromJson(request, "queryOptions");
+ if (Boolean.parseBoolean(queryOptionMap.get(
+
CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE))) {
+ return _multiStageWorkerRequestHandler.handleRequest(request,
requesterIdentity, requestContext);
+ }
}
}
return _singleStageBrokerRequestHandler.handleRequest(request,
requesterIdentity, requestContext);
diff --git
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
index dc83218d08..527e4585da 100644
---
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
+++
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
@@ -33,6 +33,8 @@ import org.testng.annotations.Test;
* Tests the various options set in the broker request
*/
public class BrokerRequestOptionsTest {
+ // TODO: remove this legacy option size checker after 0.11 release cut.
+ private static final int LEGACY_PQL_QUERY_OPTION_SIZE = 2;
@Test
public void testSetOptions() {
@@ -44,7 +46,7 @@ public class BrokerRequestOptionsTest {
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0 +
LEGACY_PQL_QUERY_OPTION_SIZE);
// TRACE
// Has trace false
@@ -52,14 +54,14 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0 +
LEGACY_PQL_QUERY_OPTION_SIZE);
// Has trace true
jsonRequest.put(Request.TRACE, true);
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get(Request.TRACE),
"true");
// DEBUG_OPTIONS (debug options will also be included as query options)
@@ -70,7 +72,6 @@ public class BrokerRequestOptionsTest {
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 1);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"),
"foo");
- Assert.assertEquals(pinotQuery.getQueryOptions(),
pinotQuery.getDebugOptions());
// Has multiple debugOptions
jsonRequest.put(Request.DEBUG_OPTIONS,
"debugOption1=foo;debugOption2=bar");
@@ -79,7 +80,6 @@ public class BrokerRequestOptionsTest {
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 2);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"),
"foo");
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption2"),
"bar");
- Assert.assertEquals(pinotQuery.getQueryOptions(),
pinotQuery.getDebugOptions());
// Invalid debug options
jsonRequest.put(Request.DEBUG_OPTIONS, "debugOption1");
@@ -100,14 +100,14 @@ public class BrokerRequestOptionsTest {
pinotQuery.setQueryOptions(queryOptions);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"),
"foo");
// Has queryOptions in query
pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo';
select * from testTable");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"),
"foo");
// Has query options in json payload
@@ -115,7 +115,7 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"),
"foo");
// Has query options in both json payload and pinotQuery, pinotQuery takes
priority
@@ -123,7 +123,7 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo';
select * from testTable;");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 2);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 2 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"),
"foo");
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption2"),
"moo");
@@ -136,7 +136,7 @@ public class BrokerRequestOptionsTest {
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query,
jsonRequest);
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 1);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"),
"foo");
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 4);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 4 +
LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"),
"bar");
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption2"),
"moo");
Assert.assertEquals(pinotQuery.getQueryOptions().get(Request.TRACE),
"true");
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
index a21279c549..a382122c33 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
@@ -29,6 +29,7 @@ import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -48,6 +49,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.helix.model.InstanceConfig;
import org.apache.pinot.common.Utils;
import org.apache.pinot.common.exception.QueryException;
+import org.apache.pinot.common.utils.config.TagNameUtils;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.access.AccessControl;
@@ -55,6 +57,7 @@ import
org.apache.pinot.controller.api.access.AccessControlFactory;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
import org.apache.pinot.spi.utils.CommonConstants;
+import
org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
@@ -104,48 +107,76 @@ public class PinotQueryResource {
}
}
- private String executeSqlQuery(@Context HttpHeaders httpHeaders, String
sqlQuery, String traceEnabled,
- String queryOptions)
- throws Exception {
- SqlNodeAndOptions sqlNodeAndOptions =
CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
- switch (sqlType) {
- case DQL:
- return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(),
traceEnabled, queryOptions, httpHeaders);
- case DML:
- Map<String, String> headers =
- httpHeaders.getRequestHeaders().entrySet().stream().filter(entry
-> !entry.getValue().isEmpty())
- .map(entry -> Pair.of(entry.getKey(), entry.getValue().get(0)))
- .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
- return _sqlQueryExecutor.executeDMLStatement(sqlNodeAndOptions,
headers).toJsonString();
- default:
- throw new UnsupportedOperationException("Unsupported SQL type - " +
sqlType);
- }
- }
-
@GET
@Path("sql")
public String handleGetSql(@QueryParam("sql") String sqlQuery,
@QueryParam("trace") String traceEnabled,
@QueryParam("queryOptions") String queryOptions, @Context HttpHeaders
httpHeaders) {
try {
LOGGER.debug("Trace: {}, Running query: {}", traceEnabled, sqlQuery);
- SqlNodeAndOptions sqlNodeAndOptions =
CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
- if (sqlType == PinotSqlType.DQL) {
- return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(),
traceEnabled, queryOptions, httpHeaders);
- }
- throw new UnsupportedOperationException("Unsupported SQL type - " +
sqlType);
+ return executeSqlQuery(httpHeaders, sqlQuery, traceEnabled,
queryOptions);
} catch (Exception e) {
LOGGER.error("Caught exception while processing get request", e);
return QueryException.getException(QueryException.INTERNAL_ERROR,
e).toString();
}
}
- public String getQueryResponse(String query, String traceEnabled, String
queryOptions, HttpHeaders httpHeaders) {
- return getQueryResponse(query, null, traceEnabled, queryOptions,
httpHeaders);
+ private String executeSqlQuery(@Context HttpHeaders httpHeaders, String
sqlQuery, String traceEnabled,
+ String queryOptions)
+ throws Exception {
+ if (queryOptions != null &&
queryOptions.contains(QueryOptionKey.USE_MULTISTAGE_ENGINE)) {
+ if
(_controllerConf.getProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED,
+ CommonConstants.Helix.DEFAULT_MULTI_STAGE_ENGINE_ENABLED)) {
+ return getMultiStageQueryResponse(sqlQuery, queryOptions, httpHeaders);
+ } else {
+ throw new UnsupportedOperationException("V2 Multi-Stage query engine
not enabled. "
+ + "Please see https://docs.pinot.apache.org/ for instruction to
enable V2 engine.");
+ }
+ } else {
+ SqlNodeAndOptions sqlNodeAndOptions =
CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
+ PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
+ switch (sqlType) {
+ case DQL:
+ return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(),
traceEnabled, queryOptions, httpHeaders);
+ case DML:
+ Map<String, String> headers =
+ httpHeaders.getRequestHeaders().entrySet().stream().filter(entry
-> !entry.getValue().isEmpty())
+ .map(entry -> Pair.of(entry.getKey(),
entry.getValue().get(0)))
+ .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+ return _sqlQueryExecutor.executeDMLStatement(sqlNodeAndOptions,
headers).toJsonString();
+ default:
+ throw new UnsupportedOperationException("Unsupported SQL type - " +
sqlType);
+ }
+ }
+ }
+
+ private String getMultiStageQueryResponse(String query, String queryOptions,
HttpHeaders httpHeaders) {
+
+ // Validate data access
+ // we don't have a cross table access control rule so only ADMIN can make
request to multi-stage engine.
+ AccessControl accessControl = _accessControlFactory.create();
+ if (!accessControl.hasAccess(httpHeaders)) {
+ return QueryException.ACCESS_DENIED_ERROR.toString();
+ }
+
+ // Get brokers, only DEFAULT tenant is supported for now.
+ // TODO: implement logic that only allows executing query where all
accessed tables are within the same tenant.
+ List<String> instanceIds = new
ArrayList<>(_pinotHelixResourceManager.getAllInstancesForBrokerTenant(
+ TagNameUtils.DEFAULT_TENANT_NAME));
+ if (instanceIds.isEmpty()) {
+ return QueryException.BROKER_RESOURCE_MISSING_ERROR.toString();
+ }
+
+ instanceIds.retainAll(_pinotHelixResourceManager.getOnlineInstanceList());
+ if (instanceIds.isEmpty()) {
+ return QueryException.BROKER_INSTANCE_MISSING_ERROR.toString();
+ }
+
+ // Send query to a random broker.
+ String instanceId = instanceIds.get(RANDOM.nextInt(instanceIds.size()));
+ return sendRequestToBroker(query, instanceId, "false", queryOptions,
httpHeaders);
}
- public String getQueryResponse(String query, @Nullable SqlNode sqlNode,
String traceEnabled, String queryOptions,
+ private String getQueryResponse(String query, @Nullable SqlNode sqlNode,
String traceEnabled, String queryOptions,
HttpHeaders httpHeaders) {
// Get resource table name.
String tableName;
@@ -180,6 +211,11 @@ public class PinotQueryResource {
// Send query to a random broker.
String instanceId = instanceIds.get(RANDOM.nextInt(instanceIds.size()));
+ return sendRequestToBroker(query, instanceId, traceEnabled, queryOptions,
httpHeaders);
+ }
+
+ private String sendRequestToBroker(String query, String instanceId, String
traceEnabled, String queryOptions,
+ HttpHeaders httpHeaders) {
InstanceConfig instanceConfig =
_pinotHelixResourceManager.getHelixInstanceConfig(instanceId);
if (instanceConfig == null) {
LOGGER.error("Instance {} not found", instanceId);
diff --git a/pinot-controller/src/main/resources/app/pages/Query.tsx
b/pinot-controller/src/main/resources/app/pages/Query.tsx
index 3ced676e92..4a6d3f3993 100644
--- a/pinot-controller/src/main/resources/app/pages/Query.tsx
+++ b/pinot-controller/src/main/resources/app/pages/Query.tsx
@@ -194,6 +194,7 @@ const QueryPage = () => {
const [checked, setChecked] = React.useState({
tracing: queryParam.get('tracing') === 'true',
+ useMSE: queryParam.get('useMSE') === 'true',
showResultJSON: false,
});
@@ -270,19 +271,24 @@ const QueryPage = () => {
setQueryLoader(true);
queryExecuted.current = true;
let params;
- let timeoutStr = '';
+ let queryOptions = '';
if(queryTimeout){
- timeoutStr = ` option(timeoutMs=${queryTimeout})`
+ queryOptions += `timeoutMs=${queryTimeout}`;
+ }
+ if(checked.useMSE){
+ queryOptions += `useMultistageEngine=true`;
}
const finalQuery = `${query || inputQuery.trim()}`;
params = JSON.stringify({
- sql: `${finalQuery}${timeoutStr}`,
+ sql: `${finalQuery}`,
trace: checked.tracing,
+ queryOptions: `${queryOptions}`,
});
if(finalQuery !== ''){
queryParam.set('query', finalQuery);
queryParam.set('tracing', checked.tracing.toString());
+ queryParam.set('useMSE', checked.useMSE.toString());
if(queryTimeout !== undefined && queryTimeout !== ''){
queryParam.set('timeout', queryTimeout.toString());
}
@@ -292,7 +298,7 @@ const QueryPage = () => {
})
}
- const results = await PinotMethodUtils.getQueryResults(params, checked);
+ const results = await PinotMethodUtils.getQueryResults(params);
setResultError(results.error || '');
setResultData(results.result || { columns: [], records: [] });
setQueryStats(results.queryStats || { columns: responseStatCols, records:
[] });
@@ -310,6 +316,10 @@ const QueryPage = () => {
+ `created with an older schema. `
+ `Please reload the table in order to refresh these segments to the
new schema.`);
}
+ if (checked.useMSE) {
+ warnings.push(`Using V2 Multi-Stage Query Engine. This is an
experimental feature. Please report any bugs to `
+ + `Apache Pinot Slack channel.`);
+ }
return warnings;
}
@@ -377,6 +387,7 @@ const QueryPage = () => {
setInputQuery(query);
setChecked({
tracing: queryParam.get('tracing') === 'true',
+ useMSE: queryParam.get('useMse') === 'true',
showResultJSON: checked.showResultJSON,
});
setQueryTimeout(Number(queryParam.get('timeout') || '') || '');
@@ -492,6 +503,16 @@ const QueryPage = () => {
Tracing
</Grid>
+ <Grid item xs={2}>
+ <Checkbox
+ name="useMSE"
+ color="primary"
+ onChange={handleChange}
+ checked={checked.useMSE}
+ />
+ Use V2 Engine
+ </Grid>
+
<Grid item xs={3}>
<FormControl fullWidth={true}
className={classes.timeoutControl}>
<InputLabel htmlFor="my-input">Timeout (in
Milliseconds)</InputLabel>
@@ -525,19 +546,20 @@ const QueryPage = () => {
</Grid>
) : null}
+ {
+ warnings.map(warn =>
+ <Alert severity="warning"
className={classes.sqlError}>
+ {warn}
+ </Alert>
+ )
+ }
+
{resultError ? (
<Alert severity="error" className={classes.sqlError}>
{resultError}
</Alert>
) : (
<>
- {
- warnings.map(warn =>
- <Alert severity="warning" className={classes.sqlError}>
- {warn}
- </Alert>
- )
- }
<Grid item xs style={{ backgroundColor: 'white' }}>
{resultData.columns.length ? (
<>
diff --git a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
index 0b6816d10b..1c41399ea0 100644
--- a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
+++ b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
@@ -254,7 +254,7 @@ const getAsObject = (str: SQLResult) => {
// This method is used to display query output in tabular format as well as
JSON format on query page
// API: /:urlName (Eg: sql or pql)
// Expected Output: {columns: [], records: []}
-const getQueryResults = (params, checkedOptions) => {
+const getQueryResults = (params) => {
return getQueryResult(params).then(({ data }) => {
let queryResponse = getAsObject(data);
@@ -268,46 +268,7 @@ const getQueryResults = (params, checkedOptions) => {
errorStr = JSON.stringify(queryResponse.exceptions, null, 2);
} else
{
- if (checkedOptions.querySyntaxPQL === true)
- {
- if (queryResponse)
- {
- if (queryResponse.selectionResults)
- {
- // Selection query
- columnList = queryResponse.selectionResults.columns;
- dataArray = queryResponse.selectionResults.results;
- }
- else if (!queryResponse.aggregationResults[0]?.groupByResult)
- {
- // Simple aggregation query
- columnList = map(queryResponse.aggregationResults,
(aggregationResult) => {
- return {title: aggregationResult.function};
- });
-
- dataArray.push(map(queryResponse.aggregationResults,
(aggregationResult) => {
- return aggregationResult.value;
- }));
- }
- else if (queryResponse.aggregationResults[0]?.groupByResult)
- {
- // Aggregation group by query
- // TODO - Revisit
- const columns = queryResponse.aggregationResults[0].groupByColumns;
- columns.push(queryResponse.aggregationResults[0].function);
- columnList = map(columns, (columnName) => {
- return columnName;
- });
-
- dataArray = map(queryResponse.aggregationResults[0].groupByResult,
(aggregationGroup) => {
- const row = aggregationGroup.group;
- row.push(aggregationGroup.value);
- return row;
- });
- }
- }
- }
- else if (queryResponse.resultTable?.dataSchema?.columnNames?.length)
+ if (queryResponse.resultTable?.dataSchema?.columnNames?.length)
{
columnList = queryResponse.resultTable.dataSchema.columnNames;
dataArray = queryResponse.resultTable.rows;
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
index e5c92c1799..60b2c4211f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
@@ -102,7 +102,8 @@ public class MultiStageEngineIntegrationTest extends
BaseClusterIntegrationTest
public void testMultiStageQuery(String sql, int expectedNumOfRows, int
expectedNumOfColumns)
throws IOException {
JsonNode multiStageResponse = JsonUtils.stringToJsonNode(
- sendPostRequest(_brokerBaseApiUrl + "/query/sql",
"{\"useMultistageEngine\": true, \"sql\":\"" + sql + "\"}"));
+ sendPostRequest(_brokerBaseApiUrl + "/query/sql",
+ "{\"queryOptions\":\"useMultistageEngine=true\", \"sql\":\"" + sql
+ "\"}"));
Assert.assertTrue(multiStageResponse.has("resultTable"));
ArrayNode jsonNode = (ArrayNode)
multiStageResponse.get("resultTable").get("rows");
// TODO: assert actual result data payload.
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
index c7179c6856..1bbb87bca5 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
@@ -93,11 +93,11 @@ public class MultistageEngineQuickStart extends
QuickStartBase {
waitForBootstrapToComplete(null);
- Map<String, String> queryOptions = Collections.singletonMap(
- CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE,
"true");
+ Map<String, String> queryOptions = Collections.singletonMap("queryOptions",
+ CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE +
"=true");
printStatus(Quickstart.Color.YELLOW, "***** Multi-stage engine quickstart
setup complete *****");
- String q1 = "SELECT count(*) FROM baseballStats_OFFLINE limit 1";
+ String q1 = "SELECT count(*) FROM baseballStats_OFFLINE";
printStatus(Quickstart.Color.YELLOW, "Total number of documents in the
table");
printStatus(Quickstart.Color.CYAN, "Query : " + q1);
printStatus(Quickstart.Color.YELLOW,
prettyPrintResponse(runner.runQuery(q1, queryOptions)));
@@ -115,8 +115,8 @@ public class MultistageEngineQuickStart extends
QuickStartBase {
printStatus(Quickstart.Color.GREEN,
"***************************************************");
printStatus(Quickstart.Color.YELLOW, "Example query run completed.");
printStatus(Quickstart.Color.GREEN,
"***************************************************");
- printStatus(Quickstart.Color.YELLOW, "Please use broker port for executing
multistage queries.");
- printStatus(Quickstart.Color.GREEN,
"***************************************************");
+ printStatus(Quickstart.Color.GREEN,
+ "You can always go to http://localhost:9000 to play around in the
query console");
}
public static void main(String[] args)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
