This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 905688f Fixing PerfBenchmarkDriver (#4109)
905688f is described below
commit 905688f9214f69770b80b2dbe5b73e8d6b6a0ea7
Author: Seunghyun Lee <[email protected]>
AuthorDate: Fri Apr 12 13:53:40 2019 -0700
Fixing PerfBenchmarkDriver (#4109)
Due to the change in #3864, controller now registers as a helix
participant. This causes the issue with PerfBenchmarkDriver
because it separately create helixZkManager and it tries to
register controller participant with the same host and port.
This pr resolves the issue.
---
.../pinot/tools/perf/PerfBenchmarkDriver.java | 47 +++++++++++++++-------
.../pinot/tools/perf/PerfBenchmarkDriverConf.java | 10 +----
2 files changed, 34 insertions(+), 23 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
index c8a6005..77e1edc 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
@@ -71,6 +71,7 @@ public class PerfBenchmarkDriver {
private final String _segmentFormatVersion;
private final boolean _verbose;
+ private ControllerStarter _controllerStarter;
private String _controllerHost;
private int _controllerPort;
private String _controllerAddress;
@@ -88,6 +89,10 @@ public class PerfBenchmarkDriver {
private final String _brokerTenantName = "DefaultTenant";
private final String _serverTenantName = "DefaultTenant";
+ // Used for creating tables and tenants, and uploading segments. Since
uncompressed segments are already available
+ // for PerfBenchmarkDriver, servers can directly load the segments.
PinotHelixResourceManager.addNewSegment(), which
+ // updates ZKSegmentMetadata only, is not exposed from controller API so we
need to update segments directly via
+ // PinotHelixResourceManager.
private PinotHelixResourceManager _helixResourceManager;
public PerfBenchmarkDriver(PerfBenchmarkDriverConf conf) {
@@ -187,7 +192,8 @@ public class PerfBenchmarkDriver {
}
ControllerConf conf = getControllerConf();
LOGGER.info("Starting controller at {}", _controllerAddress);
- new ControllerStarter(conf).start();
+ _controllerStarter = new ControllerStarter(conf);
+ _controllerStarter.start();
}
private ControllerConf getControllerConf() {
@@ -205,7 +211,7 @@ public class PerfBenchmarkDriver {
private void startBroker()
throws Exception {
- if (!_conf.isStartBroker()) {
+ if (!_conf.shouldStartBroker()) {
LOGGER.info("Skipping start broker step. Assumes broker is already
started.");
return;
}
@@ -237,18 +243,31 @@ public class PerfBenchmarkDriver {
private void startHelixResourceManager()
throws Exception {
- _helixResourceManager = new PinotHelixResourceManager(getControllerConf());
- _helixResourceManager.start();
-
- // Create broker tenant.
- Tenant brokerTenant = new
TenantBuilder(_brokerTenantName).setRole(TenantRole.BROKER).setTotalInstances(1).build();
- _helixResourceManager.createBrokerTenant(brokerTenant);
-
- // Create server tenant.
- Tenant serverTenant =
- new
TenantBuilder(_serverTenantName).setRole(TenantRole.SERVER).setTotalInstances(1).setOfflineInstances(1)
- .build();
- _helixResourceManager.createServerTenant(serverTenant);
+ if (_conf.shouldStartController()) {
+ // helix resource manager is already available at this time if
controller is started
+ _helixResourceManager = _controllerStarter.getHelixResourceManager();
+ } else {
+ // When starting server only, we need to change the controller port to
avoid registering controller helix
+ // participant with the same host and port.
+ ControllerConf controllerConf = getControllerConf();
+
controllerConf.setControllerPort(Integer.toString(_conf.getControllerPort() +
1));
+ _helixResourceManager = new PinotHelixResourceManager(controllerConf);
+ _helixResourceManager.start();
+ }
+
+ // Create server tenants if required
+ if (_conf.shouldStartServer()) {
+ Tenant serverTenant =
+ new
TenantBuilder(_serverTenantName).setRole(TenantRole.SERVER).setTotalInstances(1).setOfflineInstances(1)
+ .build();
+ _helixResourceManager.createServerTenant(serverTenant);
+ }
+
+ // Create broker tenant if required
+ if (_conf.shouldStartBroker()) {
+ Tenant brokerTenant = new
TenantBuilder(_brokerTenantName).setRole(TenantRole.BROKER).setTotalInstances(1).build();
+ _helixResourceManager.createBrokerTenant(brokerTenant);
+ }
}
private void configureResources()
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
index a994335..0f865d3 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriverConf.java
@@ -158,7 +158,7 @@ public class PerfBenchmarkDriverConf {
this.brokerHost = brokerHost;
}
- public boolean isStartBroker() {
+ public boolean shouldStartBroker() {
return startBroker;
}
@@ -194,18 +194,10 @@ public class PerfBenchmarkDriverConf {
return resultsOutputDirectory;
}
- public boolean isStartServer() {
- return startServer;
- }
-
public void setStartServer(boolean startServer) {
this.startServer = startServer;
}
- public boolean isStartController() {
- return startController;
- }
-
public void setStartController(boolean startController) {
this.startController = startController;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
