This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 737265d In Quickstart, do not start minions by default (#6749)
737265d is described below
commit 737265de3256aa56ac4731d1a3c90b374f11f0b4
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Apr 6 21:41:51 2021 -0700
In Quickstart, do not start minions by default (#6749)
---
.../pinot/tools/BatchQuickstartWithMinion.java | 5 ++
.../org/apache/pinot/tools/JoinQuickStart.java | 2 +-
.../java/org/apache/pinot/tools/Quickstart.java | 10 ++--
.../tools/admin/command/QuickstartRunner.java | 57 +++++++++++-----------
4 files changed, 41 insertions(+), 33 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
index 9dc11b6..37deaa8 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
@@ -27,6 +27,11 @@ public class BatchQuickstartWithMinion extends Quickstart {
return "examples/minions/batch/baseballStats";
}
+ @Override
+ public int getNumMinions() {
+ return 1;
+ }
+
public static void main(String[] args)
throws Exception {
PluginManager.get().init();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
index 01fc669..20ab49d 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
@@ -85,7 +85,7 @@ public class JoinQuickStart
FileUtils.copyURLToFile(resource, tableConfigFile);
QuickstartTableRequest dimTableRequest = new
QuickstartTableRequest(dimBaseballTeamsBaseDir.getAbsolutePath());
- final QuickstartRunner runner = new
QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 3, 1, 1,
baseBallStatsDataDir);
+ QuickstartRunner runner = new
QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1, 1, 3,
baseBallStatsDataDir);
printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper,
controller, broker and server *****");
runner.startAll();
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index 704550b..75e46ef 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -48,12 +48,16 @@ public class Quickstart {
return "examples/batch/baseballStats";
}
+ public int getNumMinions() {
+ return 0;
+ }
+
public String getAuthToken() {
return null;
}
public Map<String, Object> getConfigOverrides() {
- return new HashMap<>();
+ return null;
}
public static void printStatus(Color color, String message) {
@@ -172,8 +176,8 @@ public class Quickstart {
FileUtils.copyURLToFile(resource, tableConfigFile);
QuickstartTableRequest request = new
QuickstartTableRequest(baseDir.getAbsolutePath());
- final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir,
true, getAuthToken(),
+ QuickstartRunner runner =
+ new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1,
getNumMinions(), dataDir, true, getAuthToken(),
getConfigOverrides());
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and
server *****");
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index 2fc3376..f3ae041 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -57,46 +57,39 @@ public class QuickstartRunner {
private static final int ZK_PORT = 2123;
private static final String ZK_ADDRESS = "localhost:" + ZK_PORT;
- private static final int DEFAULT_SERVER_NETTY_PORT = 7000;
- private static final int DEFAULT_SERVER_ADMIN_API_PORT = 7500;
- private static final int DEFAULT_BROKER_PORT = 8000;
private static final int DEFAULT_CONTROLLER_PORT = 9000;
+ private static final int DEFAULT_BROKER_PORT = 8000;
+ private static final int DEFAULT_SERVER_ADMIN_API_PORT = 7500;
+ private static final int DEFAULT_SERVER_NETTY_PORT = 7000;
private static final int DEFAULT_MINION_PORT = 6000;
-
private static final String DEFAULT_ZK_DIR = "PinotZkDir";
private static final String DEFAULT_CONTROLLER_DIR = "PinotControllerDir";
private static final String DEFAULT_SERVER_DATA_DIR = "PinotServerDataDir";
private static final String DEFAULT_SERVER_SEGMENT_DIR =
"PinotServerSegmentDir";
private final List<QuickstartTableRequest> _tableRequests;
- private final int _numServers;
- private final int _numBrokers;
private final int _numControllers;
+ private final int _numBrokers;
+ private final int _numServers;
private final int _numMinions;
private final File _tempDir;
private final boolean _enableTenantIsolation;
private final String _authToken;
private final Map<String, Object> _configOverrides;
- private final List<Integer> _brokerPorts = new ArrayList<>();
private final List<Integer> _controllerPorts = new ArrayList<>();
- private final List<String> _segmentDirs = new ArrayList<>();
+ private final List<Integer> _brokerPorts = new ArrayList<>();
private boolean _isStopped = false;
- public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numServers, int numBrokers,
- int numControllers, File tempDir, boolean enableIsolation)
- throws Exception {
- this(tableRequests, numServers, numBrokers, numControllers, 1, tempDir,
enableIsolation, null, null);
- }
-
- public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numServers, int numBrokers,
- int numControllers, int numMinions, File tempDir, boolean
enableIsolation, String authToken, Map<String, Object> configOverrides)
+ public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numControllers, int numBrokers,
+ int numServers, int numMinions, File tempDir, boolean enableIsolation,
String authToken,
+ Map<String, Object> configOverrides)
throws Exception {
_tableRequests = tableRequests;
- _numServers = numServers;
- _numBrokers = numBrokers;
_numControllers = numControllers;
+ _numBrokers = numBrokers;
+ _numServers = numServers;
_numMinions = numMinions;
_tempDir = tempDir;
_enableTenantIsolation = enableIsolation;
@@ -105,10 +98,10 @@ public class QuickstartRunner {
clean();
}
- public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numServers, int numBrokers,
- int numControllers, File tempDir)
+ public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numControllers, int numBrokers,
+ int numServers, File tempDir)
throws Exception {
- this(tableRequests, numServers, numBrokers, numControllers, 1, tempDir,
true, null, null);
+ this(tableRequests, numControllers, numBrokers, numServers, 0, tempDir,
true, null, null);
}
private void startZookeeper()
@@ -125,7 +118,8 @@ public class QuickstartRunner {
StartControllerCommand controllerStarter = new StartControllerCommand();
controllerStarter.setControllerPort(String.valueOf(DEFAULT_CONTROLLER_PORT +
i)).setZkAddress(ZK_ADDRESS)
.setClusterName(CLUSTER_NAME).setTenantIsolation(_enableTenantIsolation)
- .setDataDir(new File(_tempDir, DEFAULT_CONTROLLER_DIR +
i).getAbsolutePath()).setConfigOverrides(_configOverrides);
+ .setDataDir(new File(_tempDir, DEFAULT_CONTROLLER_DIR +
i).getAbsolutePath())
+ .setConfigOverrides(_configOverrides);
controllerStarter.execute();
_controllerPorts.add(DEFAULT_CONTROLLER_PORT + i);
}
@@ -135,7 +129,8 @@ public class QuickstartRunner {
throws Exception {
for (int i = 0; i < _numBrokers; i++) {
StartBrokerCommand brokerStarter = new StartBrokerCommand();
- brokerStarter.setPort(DEFAULT_BROKER_PORT +
i).setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME).setConfigOverrides(_configOverrides);
+ brokerStarter.setPort(DEFAULT_BROKER_PORT +
i).setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME)
+ .setConfigOverrides(_configOverrides);
brokerStarter.execute();
_brokerPorts.add(DEFAULT_BROKER_PORT + i);
}
@@ -148,7 +143,8 @@ public class QuickstartRunner {
serverStarter.setPort(DEFAULT_SERVER_NETTY_PORT +
i).setAdminPort(DEFAULT_SERVER_ADMIN_API_PORT + i)
.setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME)
.setDataDir(new File(_tempDir, DEFAULT_SERVER_DATA_DIR +
i).getAbsolutePath())
- .setSegmentDir(new File(_tempDir, DEFAULT_SERVER_SEGMENT_DIR +
i).getAbsolutePath()).setConfigOverrides(_configOverrides);
+ .setSegmentDir(new File(_tempDir, DEFAULT_SERVER_SEGMENT_DIR +
i).getAbsolutePath())
+ .setConfigOverrides(_configOverrides);
serverStarter.execute();
}
}
@@ -157,8 +153,8 @@ public class QuickstartRunner {
throws Exception {
for (int i = 0; i < _numMinions; i++) {
StartMinionCommand minionStarter = new StartMinionCommand();
- minionStarter.setMinionPort(DEFAULT_MINION_PORT + i)
-
.setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME).setConfigOverrides(_configOverrides);
+ minionStarter.setMinionPort(DEFAULT_MINION_PORT +
i).setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME)
+ .setConfigOverrides(_configOverrides);
minionStarter.execute();
}
}
@@ -184,6 +180,7 @@ public class QuickstartRunner {
return;
}
+ // TODO: Stop Minion
StopProcessCommand stopper = new StopProcessCommand(false);
stopper.stopController().stopBroker().stopServer().stopZookeeper();
stopper.execute();
@@ -258,7 +255,8 @@ public class QuickstartRunner {
}
public static void registerDefaultPinotFS() {
- registerPinotFS("s3", "org.apache.pinot.plugin.filesystem.S3PinotFS",
ImmutableMap.of("region", System.getProperty("AWS_REGION", "us-west-2")));
+ registerPinotFS("s3", "org.apache.pinot.plugin.filesystem.S3PinotFS",
+ ImmutableMap.of("region", System.getProperty("AWS_REGION",
"us-west-2")));
}
public static void registerPinotFS(String scheme, String fsClassName,
Map<String, Object> configs) {
@@ -270,7 +268,8 @@ public class QuickstartRunner {
PinotFSFactory.register(scheme, fsClassName, new
PinotConfiguration(configs));
LOGGER.info("Registered PinotFS for scheme: {}", scheme);
} catch (Exception e) {
- LOGGER.info("Unable to init PinotFS for scheme: {}, class name: {},
configs: {}, Error: {}", scheme, fsClassName, configs, e);
+ LOGGER
+ .error("Unable to init PinotFS for scheme: {}, class name: {},
configs: {}", scheme, fsClassName, configs, e);
}
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
