[
https://issues.apache.org/jira/browse/HIVE-26443?focusedWorklogId=805235&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-805235
]
ASF GitHub Bot logged work on HIVE-26443:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 31/Aug/22 13:51
Start Date: 31/Aug/22 13:51
Worklog Time Spent: 10m
Work Description: deniskuzZ commented on code in PR #3513:
URL: https://github.com/apache/hive/pull/3513#discussion_r959614105
##########
service/src/java/org/apache/hive/service/server/HiveServer2.java:
##########
@@ -1140,11 +1146,59 @@ private static void startHiveServer2() throws Throwable
{
private void maybeStartCompactorThreads(HiveConf hiveConf) throws Exception {
if (MetastoreConf.getVar(hiveConf,
MetastoreConf.ConfVars.HIVE_METASTORE_RUNWORKER_IN).equals("hs2")) {
int numWorkers = MetastoreConf.getIntVar(hiveConf,
MetastoreConf.ConfVars.COMPACTOR_WORKER_THREADS);
- for (int i = 0; i < numWorkers; i++) {
- Worker w = new Worker();
- CompactorThread.initializeAndStartThread(w, hiveConf);
+ List<RegexMatchingConfEntry> entries =
hiveConf.getMatchingEntries(Constants.COMPACTION_POOLS_PATTERN);
+
+ StringBuilder sb = new StringBuilder(2048);
+ sb.append("This HS2 instance will act as Compactor with the following
worker pool configuration:\n");
+ sb.append("Global pool size: ").append(numWorkers).append("\n");
+
+ LOG.info("Initializing the compaction pools with using the global worker
limit: {} ", numWorkers);
+ while (numWorkers > 0 && entries.size() > 0) {
+ RegexMatchingConfEntry entry = entries.remove(0);
+ String poolName = entry.getMatcher().group(0);
+ int poolWorkers = hiveConf.getInt(entry.getKey(), 0);
+
+ if (poolWorkers == 0) {
+ LOG.warn("Compaction pool ({}) configured with zero workers.
Skipping pool initialization", poolName);
+ sb.append("Pool not initialized, 0 size:
").append(poolName).append("\n");
+ continue;
+ }
+ if (poolWorkers > numWorkers) {
+ LOG.warn("Global worker pool exhausted, compaction pool ({}) will be
configured with less workers than the " +
+ "required number. ({} -> {})", poolName, poolWorkers,
numWorkers);
+ poolWorkers = numWorkers;
+ }
+
+ LOG.info("Initializing compaction pool ({}) with {} workers.",
poolName, poolWorkers);
+ for (int i = 0; i < poolWorkers; i++) {
+ Worker w = new Worker();
+ w.setPoolName(poolName);
+ CompactorThread.initializeAndStartThread(w, hiveConf);
+ sb.append("Worker - Name: ").append(w.getName()).append(", Pool:
").append(poolName)
+ .append(", Priority: ").append(w.getPriority()).append("\n");
+ }
+ numWorkers -= poolWorkers;
+ }
+
+ if (numWorkers == 0) {
+ LOG.warn("No default compaction pool configured, all non-labeled
compaction requests will remain unprocessed!");
+ if (entries.size() > 0) {
+ for(RegexMatchingConfEntry entry : entries) {
Review Comment:
nit: missing space
Issue Time Tracking
-------------------
Worklog Id: (was: 805235)
Time Spent: 10.5h (was: 10h 20m)
> Add priority queueing to compaction
> -----------------------------------
>
> Key: HIVE-26443
> URL: https://issues.apache.org/jira/browse/HIVE-26443
> Project: Hive
> Issue Type: New Feature
> Reporter: László Végh
> Assignee: László Végh
> Priority: Major
> Labels: pull-request-available
> Attachments: Pool based compaction queues.docx
>
> Time Spent: 10.5h
> Remaining Estimate: 0h
>
> The details can be found in the attached design doc.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
