deniskuzZ commented on code in PR #3513:
URL: https://github.com/apache/hive/pull/3513#discussion_r959614105
##########
service/src/java/org/apache/hive/service/server/HiveServer2.java:
##########
@@ -1140,11 +1146,59 @@ private static void startHiveServer2() throws Throwable
{
private void maybeStartCompactorThreads(HiveConf hiveConf) throws Exception {
if (MetastoreConf.getVar(hiveConf,
MetastoreConf.ConfVars.HIVE_METASTORE_RUNWORKER_IN).equals("hs2")) {
int numWorkers = MetastoreConf.getIntVar(hiveConf,
MetastoreConf.ConfVars.COMPACTOR_WORKER_THREADS);
- for (int i = 0; i < numWorkers; i++) {
- Worker w = new Worker();
- CompactorThread.initializeAndStartThread(w, hiveConf);
+ List<RegexMatchingConfEntry> entries =
hiveConf.getMatchingEntries(Constants.COMPACTION_POOLS_PATTERN);
+
+ StringBuilder sb = new StringBuilder(2048);
+ sb.append("This HS2 instance will act as Compactor with the following
worker pool configuration:\n");
+ sb.append("Global pool size: ").append(numWorkers).append("\n");
+
+ LOG.info("Initializing the compaction pools with using the global worker
limit: {} ", numWorkers);
+ while (numWorkers > 0 && entries.size() > 0) {
+ RegexMatchingConfEntry entry = entries.remove(0);
+ String poolName = entry.getMatcher().group(0);
+ int poolWorkers = hiveConf.getInt(entry.getKey(), 0);
+
+ if (poolWorkers == 0) {
+ LOG.warn("Compaction pool ({}) configured with zero workers.
Skipping pool initialization", poolName);
+ sb.append("Pool not initialized, 0 size:
").append(poolName).append("\n");
+ continue;
+ }
+ if (poolWorkers > numWorkers) {
+ LOG.warn("Global worker pool exhausted, compaction pool ({}) will be
configured with less workers than the " +
+ "required number. ({} -> {})", poolName, poolWorkers,
numWorkers);
+ poolWorkers = numWorkers;
+ }
+
+ LOG.info("Initializing compaction pool ({}) with {} workers.",
poolName, poolWorkers);
+ for (int i = 0; i < poolWorkers; i++) {
+ Worker w = new Worker();
+ w.setPoolName(poolName);
+ CompactorThread.initializeAndStartThread(w, hiveConf);
+ sb.append("Worker - Name: ").append(w.getName()).append(", Pool:
").append(poolName)
+ .append(", Priority: ").append(w.getPriority()).append("\n");
+ }
+ numWorkers -= poolWorkers;
+ }
+
+ if (numWorkers == 0) {
+ LOG.warn("No default compaction pool configured, all non-labeled
compaction requests will remain unprocessed!");
+ if (entries.size() > 0) {
+ for(RegexMatchingConfEntry entry : entries) {
Review Comment:
nit: missing space
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
