This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 43e785dfb3 Reuse broker/server hostname for multistage queryrunner
hostname (#11473)
43e785dfb3 is described below
commit 43e785dfb31d388bbb3bc3dda36ee4016aa8868f
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Sep 1 05:56:12 2023 -0700
Reuse broker/server hostname for multistage queryrunner hostname (#11473)
---
.../broker/broker/helix/BaseBrokerStarter.java | 4 +++
.../apache/pinot/query/QueryServerEnclosure.java | 3 +--
.../pinot/query/runtime/QueryRunnerTest.java | 30 +++++++++++++---------
.../runtime/queries/ResourceBasedQueriesTest.java | 6 ++---
.../server/starter/helix/BaseServerStarter.java | 4 +++
.../apache/pinot/spi/utils/CommonConstants.java | 1 -
6 files changed, 29 insertions(+), 19 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
index 349f01e2b1..c6b3627536 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
@@ -146,6 +146,10 @@ public abstract class BaseBrokerStarter implements
ServiceStartable {
_brokerConf.getProperty(Helix.SET_INSTANCE_ID_TO_HOSTNAME_KEY,
false) ? NetUtils.getHostnameOrAddress()
: NetUtils.getHostAddress();
}
+ // Override multi-stage query runner hostname if not set explicitly
+ if
(!_brokerConf.containsKey(MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME)) {
+
_brokerConf.setProperty(MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
_hostname);
+ }
_port = _listenerConfigs.get(0).getPort();
_tlsPort = ListenerConfigUtil.findLastTlsPort(_listenerConfigs, -1);
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryServerEnclosure.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryServerEnclosure.java
index 738c53f53f..fd7903c1b9 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryServerEnclosure.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryServerEnclosure.java
@@ -77,8 +77,7 @@ public class QueryServerEnclosure {
_helixManager = mockHelixManager(factory.buildSchemaMap());
_queryRunnerPort = QueryTestUtils.getAvailablePort();
_runnerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_PORT,
_queryRunnerPort);
-
_runnerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
- String.format("Server_%s",
CommonConstants.MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_HOSTNAME));
+
_runnerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
"Server_localhost");
_queryRunner = new QueryRunner();
} catch (Exception e) {
throw new RuntimeException("Test Failed!", e);
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
index afe5aa34ab..076b565743 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
@@ -128,14 +128,12 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
}
_reducerGrpcPort = QueryTestUtils.getAvailablePort();
- _reducerHostname = String.format("Broker_%s",
CommonConstants.MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_HOSTNAME);
+ _reducerHostname = "Broker_localhost";
Map<String, Object> reducerConfig = new HashMap<>();
reducerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_PORT,
_reducerGrpcPort);
reducerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
_reducerHostname);
_reducerScheduler = new OpChainSchedulerService(EXECUTOR);
- _mailboxService =
- new
MailboxService(CommonConstants.MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_HOSTNAME,
_reducerGrpcPort,
- new PinotConfiguration(reducerConfig));
+ _mailboxService = new MailboxService("localhost", _reducerGrpcPort, new
PinotConfiguration(reducerConfig));
_mailboxService.start();
_queryEnvironment =
@@ -277,15 +275,23 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
// test groups limit in both leaf and intermediate stage
new Object[]{"SET numGroupsLimit = 1; SELECT col1, COUNT(*) FROM a
GROUP BY col1", 1},
new Object[]{"SET numGroupsLimit = 2; SELECT col1, COUNT(*) FROM a
GROUP BY col1", 2},
- new Object[]{"SET numGroupsLimit = 1; "
- + "SELECT a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1)
GROUP BY a.col2, b.col2", 1},
- new Object[]{"SET numGroupsLimit = 2; "
- + "SELECT a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1)
GROUP BY a.col2, b.col2", 2},
+ new Object[]{
+ "SET numGroupsLimit = 1; "
+ + "SELECT a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1)
GROUP BY a.col2, b.col2", 1
+ },
+ new Object[]{
+ "SET numGroupsLimit = 2; "
+ + "SELECT a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1)
GROUP BY a.col2, b.col2", 2
+ },
// TODO: Consider pushing down hint to the leaf stage
- new Object[]{"SET numGroupsLimit = 2; SELECT /*+
aggOptions(num_groups_limit='1') */ "
- + "col1, COUNT(*) FROM a GROUP BY col1", 2},
- new Object[]{"SET numGroupsLimit = 2; SELECT /*+
aggOptions(num_groups_limit='1') */ "
- + "a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1) GROUP BY
a.col2, b.col2", 1}
+ new Object[]{
+ "SET numGroupsLimit = 2; SELECT /*+
aggOptions(num_groups_limit='1') */ "
+ + "col1, COUNT(*) FROM a GROUP BY col1", 2
+ },
+ new Object[]{
+ "SET numGroupsLimit = 2; SELECT /*+
aggOptions(num_groups_limit='1') */ "
+ + "a.col2, b.col2, COUNT(*) FROM a JOIN b USING (col1) GROUP
BY a.col2, b.col2", 1
+ }
};
}
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
index 38f03534dd..fc29c7c235 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
@@ -184,14 +184,12 @@ public class ResourceBasedQueriesTest extends
QueryRunnerTestBase {
QueryServerEnclosure server2 = new QueryServerEnclosure(factory2);
_reducerGrpcPort = QueryTestUtils.getAvailablePort();
- _reducerHostname = String.format("Broker_%s",
CommonConstants.MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_HOSTNAME);
+ _reducerHostname = "Broker_localhost";
Map<String, Object> reducerConfig = new HashMap<>();
reducerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_PORT,
_reducerGrpcPort);
reducerConfig.put(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
_reducerHostname);
_reducerScheduler = new OpChainSchedulerService(EXECUTOR);
- _mailboxService =
- new
MailboxService(CommonConstants.MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_HOSTNAME,
_reducerGrpcPort,
- new PinotConfiguration(reducerConfig));
+ _mailboxService = new MailboxService("localhost", _reducerGrpcPort, new
PinotConfiguration(reducerConfig));
_mailboxService.start();
Map<String, List<String>> tableToSegmentMap1 =
factory1.buildTableSegmentNameMap();
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
index 2d97d9b93f..dac34e210b 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
@@ -154,6 +154,10 @@ public abstract class BaseServerStarter implements
ServiceStartable {
_hostname = _serverConf.getProperty(Helix.KEY_OF_SERVER_NETTY_HOST,
_serverConf.getProperty(Helix.SET_INSTANCE_ID_TO_HOSTNAME_KEY, false)
? NetUtils.getHostnameOrAddress()
: NetUtils.getHostAddress());
+ // Override multi-stage query runner hostname if not set explicitly
+ if
(!_serverConf.containsKey(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME))
{
+
_serverConf.setProperty(CommonConstants.MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_HOSTNAME,
_hostname);
+ }
_port = _serverConf.getProperty(Helix.KEY_OF_SERVER_NETTY_PORT,
Helix.DEFAULT_SERVER_NETTY_PORT);
_instanceId = _serverConf.getProperty(Server.CONFIG_OF_INSTANCE_ID);
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 3f0ab0d2b6..6b071e8f51 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1050,7 +1050,6 @@ public class CommonConstants {
* {@link org.apache.pinot.common.datablock.DataBlock}.
*/
public static final String KEY_OF_QUERY_RUNNER_HOSTNAME =
"pinot.query.runner.hostname";
- public static final String DEFAULT_QUERY_RUNNER_HOSTNAME = "localhost";
public static final String KEY_OF_QUERY_RUNNER_PORT =
"pinot.query.runner.port";
public static final int DEFAULT_QUERY_RUNNER_PORT = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
