hive git commit: HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

2016-08-09 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 39a0e5509 -> 6b27b6a64


HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is 
disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

(cherry picked from commit f1d3553728d772b1db68d34066bd66864c2e66e5)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b27b6a6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b27b6a6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b27b6a6

Branch: refs/heads/branch-2.1
Commit: 6b27b6a64d65632e8369d75b4a53213d72a99186
Parents: 39a0e55
Author: Siddharth Seth 
Authored: Tue Aug 9 15:40:21 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Aug 9 15:41:15 2016 -0700

--
 .../hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java   | 3 +++
 .../hive/llap/tezplugins/TestLlapTaskSchedulerService.java  | 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6b27b6a6/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 6fa3107..efd774d 100644
--- 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -870,6 +870,9 @@ public class LlapTaskSchedulerService extends TaskScheduler 
{
 if (metrics != null) {
   metrics.setDisabledNodeCount(disabledNodesQueue.size());
 }
+// Trigger a scheduling run - in case there's some task which was 
waiting for this node to
+// become available.
+trySchedulingPendingTasks();
   }
 } finally {
   writeLock.unlock();

http://git-wip-us.apache.org/repos/asf/hive/blob/6b27b6a6/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
index afbab95..402658b 100644
--- 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
+++ 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -773,7 +773,7 @@ public class TestLlapTaskSchedulerService {
   // At this point. one p=2 task and task3(p=1) running. Ask for another 
p1 task.
   while (true) {
 tsWrapper.signalSchedulerRun();
-tsWrapper.awaitSchedulerRun();
+tsWrapper.awaitSchedulerRun(1000l);
 if (tsWrapper.ts.dagStats.numPreemptedTasks == 2) {
   break;
 }
@@ -1174,6 +1174,9 @@ public class TestLlapTaskSchedulerService {
   assertEquals(task2, argumentCaptor.getAllValues().get(1));
 
   reset(tsWrapper.mockAppCallback);
+  // Flush any pending scheduler runs which may be blocked. Wait 2 seconds 
for the run to complete.
+  tsWrapper.signalSchedulerRun();
+  tsWrapper.awaitSchedulerRun(2000l);
 
   // Mark a task as failed due to a comm failure.
   tsWrapper.deallocateTask(task1, false, 
TaskAttemptEndReason.COMMUNICATION_ERROR);

hive git commit: HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

2016-08-09 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/master 2ef3ab855 -> f1d355372


HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is 
disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f1d35537
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f1d35537
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f1d35537

Branch: refs/heads/master
Commit: f1d3553728d772b1db68d34066bd66864c2e66e5
Parents: 2ef3ab8
Author: Siddharth Seth 
Authored: Tue Aug 9 15:40:21 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Aug 9 15:40:21 2016 -0700

--
 .../hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java   | 3 +++
 .../hive/llap/tezplugins/TestLlapTaskSchedulerService.java  | 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f1d35537/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 6fa3107..efd774d 100644
--- 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -870,6 +870,9 @@ public class LlapTaskSchedulerService extends TaskScheduler 
{
 if (metrics != null) {
   metrics.setDisabledNodeCount(disabledNodesQueue.size());
 }
+// Trigger a scheduling run - in case there's some task which was 
waiting for this node to
+// become available.
+trySchedulingPendingTasks();
   }
 } finally {
   writeLock.unlock();

http://git-wip-us.apache.org/repos/asf/hive/blob/f1d35537/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
index afbab95..402658b 100644
--- 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
+++ 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -773,7 +773,7 @@ public class TestLlapTaskSchedulerService {
   // At this point. one p=2 task and task3(p=1) running. Ask for another 
p1 task.
   while (true) {
 tsWrapper.signalSchedulerRun();
-tsWrapper.awaitSchedulerRun();
+tsWrapper.awaitSchedulerRun(1000l);
 if (tsWrapper.ts.dagStats.numPreemptedTasks == 2) {
   break;
 }
@@ -1174,6 +1174,9 @@ public class TestLlapTaskSchedulerService {
   assertEquals(task2, argumentCaptor.getAllValues().get(1));
 
   reset(tsWrapper.mockAppCallback);
+  // Flush any pending scheduler runs which may be blocked. Wait 2 seconds 
for the run to complete.
+  tsWrapper.signalSchedulerRun();
+  tsWrapper.awaitSchedulerRun(2000l);
 
   // Mark a task as failed due to a comm failure.
   tsWrapper.deallocateTask(task1, false, 
TaskAttemptEndReason.COMMUNICATION_ERROR);