This is an automated email from the ASF dual-hosted git repository. prasanthj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new b5de84a HIVE-22988: LLAP: If consistent splits is disabled ordering instances is not required (Prasanth Jayachandran reviewed by Slim Bouguerra) b5de84a is described below commit b5de84a6ce27904562bb15d7fe800b6ae3c32670 Author: Prasanth Jayachandran <prasan...@apache.org> AuthorDate: Mon Mar 9 21:42:33 2020 -0700 HIVE-22988: LLAP: If consistent splits is disabled ordering instances is not required (Prasanth Jayachandran reviewed by Slim Bouguerra) --- .../hive/llap/tezplugins/LlapTaskSchedulerService.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java index d11bf13..48501e5 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java @@ -296,6 +296,7 @@ public class LlapTaskSchedulerService extends TaskScheduler { private int totalGuaranteed = 0, unusedGuaranteed = 0; + private final boolean consistentSplits; /** * An internal version to make sure we don't race and overwrite a newer totalGuaranteed count in * ZK with an older one, without requiring us to make ZK updates under the main writeLock. @@ -345,6 +346,7 @@ public class LlapTaskSchedulerService extends TaskScheduler { LOCK_METRICS); readLock = lock.readLock(); writeLock = lock.writeLock(); + this.consistentSplits = HiveConf.getBoolVar(conf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS); if (conf.getBoolean(LLAP_PLUGIN_ENDPOINT_ENABLED, false)) { JobTokenSecretManager sm = null; @@ -444,8 +446,8 @@ public class LlapTaskSchedulerService extends TaskScheduler { String hostsString = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); LOG.info("Running with configuration: hosts={}, numSchedulableTasksPerNode={}, " - + "nodeBlacklistConf={}, localityConf={}", - hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf); + + "nodeBlacklistConf={}, localityConf={} consistentSplits={}", + hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf, consistentSplits); this.amRegistry = TezAmRegistryImpl.create(conf, true); synchronized (LlapTaskCommunicator.pluginInitLock) { @@ -1476,7 +1478,13 @@ public class LlapTaskSchedulerService extends TaskScheduler { } /* fall through - miss in locality or no locality-requested */ - Collection<LlapServiceInstance> instances = activeInstances.getAllInstancesOrdered(true); + Collection<LlapServiceInstance> instances; + if (consistentSplits) { + instances = activeInstances.getAllInstancesOrdered(true); + } else { + // if consistent splits are not used we don't need the ordering as there will be no cache benefit anyways + instances = activeInstances.getAll(); + } List<NodeInfo> allNodes = new ArrayList<>(instances.size()); List<NodeInfo> activeNodesWithFreeSlots = new ArrayList<>(); for (LlapServiceInstance inst : instances) {