[
https://issues.apache.org/jira/browse/HIVE-21909?focusedWorklogId=271672&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-271672
]
ASF GitHub Bot logged work on HIVE-21909:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Jul/19 14:58
Start Date: 03/Jul/19 14:58
Worklog Time Spent: 10m
Work Description: asinkovits commented on pull request #695: HIVE-21909:
Publish the LLAP Daemon capacity through ZooKeeper and ho…
URL: https://github.com/apache/hive/pull/695#discussion_r300005114
##########
File path:
llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
##########
@@ -0,0 +1,124 @@
+package org.apache.hadoop.hive.llap.registry.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryOneTime;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
+import org.apache.hadoop.hive.registry.ServiceInstanceSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
+import org.mockito.internal.util.reflection.Fields;
+import org.mockito.internal.util.reflection.InstanceField;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.lang.Integer.parseInt;
+import static org.junit.Assert.assertEquals;
+
+public class TestLlapZookeeperRegistryImpl {
+
+ @Spy
+ private HiveConf mockConf = new HiveConf();
+
+ private LlapZookeeperRegistryImpl registry;
+
+ private CuratorFramework curatorFramework;
+ private TestingServer server;
+
+ @Before
+ public void setUp() throws Exception {
+ MockitoAnnotations.initMocks(this);
+ registry = new LlapZookeeperRegistryImpl("TestLlapZookeeperRegistryImpl",
mockConf);
+
+ server = new TestingServer();
+ server.start();
+
+ curatorFramework = CuratorFrameworkFactory.
+ builder().
+ connectString(server.getConnectString()).
+ sessionTimeoutMs(1000).
+ retryPolicy(new RetryOneTime(1000)).
+ build();
+ curatorFramework.start();
+
+ trySetMock(registry, CuratorFramework.class, curatorFramework);
+
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ curatorFramework.close();
+ server.stop();
+ }
+
+ @Test
+ public void testRegister() throws Exception {
+ // Given
+ int expectedExecutorCount = HiveConf.getIntVar(mockConf,
HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+ int expectedQueueSize = HiveConf.getIntVar(mockConf,
HiveConf.ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE);
+
+ // When
+ registry.register();
+ ServiceInstanceSet<LlapServiceInstance> serviceInstanceSet =
+ registry.getInstances("LLAP", 1000);
+
+ // Then
+ Collection<LlapServiceInstance> llaps = serviceInstanceSet.getAll();
+ assertEquals(1, llaps.size());
+ LlapServiceInstance serviceInstance = llaps.iterator().next();
+ Map<String, String> attributes = serviceInstance.getProperties();
+
+ assertEquals(expectedQueueSize,
+
parseInt(attributes.get(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE)));
+ assertEquals(expectedExecutorCount,
+
parseInt(attributes.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS)));
+ }
+
+ @Test
+ public void testUpdate() throws Exception {
+ // Given
+ String expectedExecutorCount = "2";
+ String expectedQueueSize = "20";
+ Map<String, String> capacityValues = new HashMap<>(2);
+ capacityValues.put(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS,
expectedExecutorCount);
+
capacityValues.put(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE,
expectedQueueSize);
+
+ // When
+ registry.register();
+ registry.updateRegistration(capacityValues.entrySet());
+ ServiceInstanceSet<LlapServiceInstance> serviceInstanceSet =
+ registry.getInstances("LLAP", 1000);
+
+ // Then
+ Collection<LlapServiceInstance> llaps = serviceInstanceSet.getAll();
+ assertEquals(1, llaps.size());
+ LlapServiceInstance serviceInstance = llaps.iterator().next();
+ Map<String, String> attributes = serviceInstance.getProperties();
+
+ assertEquals(expectedQueueSize,
+
attributes.get(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE));
+ assertEquals(expectedExecutorCount,
+
attributes.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS));
+ }
+
+ static <T> void trySetMock(Object o, Class<T> clazz, T mock) {
+ List<InstanceField> instanceFields = Fields
+ .allDeclaredFieldsOf(o)
+ .filter(instanceField ->
!clazz.isAssignableFrom(instanceField.jdkField().getType()))
+ .instanceFields();
+ if (instanceFields.size() != 1) {
Review comment:
It would be easier, but I think this way it's more robust.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 271672)
Time Spent: 1h 40m (was: 1.5h)
> Publish the LLAP Daemon capacity through ZooKeeper and honor the capacity
> when scheduling new tasks
> ---------------------------------------------------------------------------------------------------
>
> Key: HIVE-21909
> URL: https://issues.apache.org/jira/browse/HIVE-21909
> Project: Hive
> Issue Type: Sub-task
> Components: llap
> Reporter: Peter Vary
> Assignee: Antal Sinkovits
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-21909.01.patch
>
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> The changes in LlapDaemonNode status (or executor number) should be reflected
> in the ZooKeeper information, and we should make sure TezAMs are reading and
> honor the changes.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
