hive git commit: HIVE-14436: Hive 1.2.1/Hitting "ql.Driver: FAILED: IllegalArgumentException Error: , expected at the end of 'decimal(9'" after enabling hive.optimize.skewjoin and with MR engine (Dani

2016-08-09 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/branch-1 996a16ca6 -> 20908265e


HIVE-14436: Hive 1.2.1/Hitting "ql.Driver: FAILED: IllegalArgumentException 
Error: , expected at the end of 'decimal(9'" after enabling 
hive.optimize.skewjoin and with MR engine (Daniel Dai, reviewed by Ashutosh 
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/20908265
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/20908265
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/20908265

Branch: refs/heads/branch-1
Commit: 20908265e2b3e303125f03623e6018f28806d65a
Parents: 996a16c
Author: Daniel Dai 
Authored: Tue Aug 9 22:57:30 2016 -0700
Committer: Daniel Dai 
Committed: Tue Aug 9 22:57:30 2016 -0700

--
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 10 +--
 .../queries/clientpositive/decimal_skewjoin.q   | 21 ++
 .../clientpositive/decimal_skewjoin.q.out   | 77 
 3 files changed, 101 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/20908265/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 0a32e6c..bfe8657 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2509,13 +2509,9 @@ public final class Utilities {
   public static List getColumnTypes(Properties props) {
 List names = new ArrayList();
 String colNames = props.getProperty(serdeConstants.LIST_COLUMN_TYPES);
-String[] cols = colNames.trim().split(",");
-if (cols != null) {
-  for (String col : cols) {
-if (col != null && !col.trim().equals("")) {
-  names.add(col);
-}
-  }
+ArrayList cols = 
TypeInfoUtils.getTypeInfosFromTypeString(colNames);
+for (TypeInfo col : cols) {
+  names.add(col.getTypeName());
 }
 return names;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/20908265/ql/src/test/queries/clientpositive/decimal_skewjoin.q
--
diff --git a/ql/src/test/queries/clientpositive/decimal_skewjoin.q 
b/ql/src/test/queries/clientpositive/decimal_skewjoin.q
new file mode 100644
index 000..cac0d89
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_skewjoin.q
@@ -0,0 +1,21 @@
+
+set hive.execution.engine=mr;
+set hive.optimize.skewjoin=true;
+set hive.auto.convert.join=false;
+
+drop table if exists decimal_skewjoin_1;
+drop table if exists decimal_skewjoin_2;
+
+create table decimal_skewjoin_1 (t decimal(4,2), u decimal(5), v decimal);
+create table decimal_skewjoin_2 (t decimal(4,2), u decimal(5), v decimal);
+
+insert overwrite table decimal_skewjoin_1
+  select cast('17.29' as decimal(4,2)), 3.1415926BD, 3115926.54321BD from src 
tablesample (1 rows);
+
+insert overwrite table decimal_skewjoin_2
+  select cast('17.29' as decimal(4,2)), 3.1415926BD, 3115926.54321BD from src 
tablesample (1 rows);
+
+select a.u from decimal_skewjoin_1 a INNER JOIN decimal_skewjoin_2 b ON 
a.t=b.t;
+
+drop table decimal_skewjoin_1;
+drop table decimal_skewjoin_2;

http://git-wip-us.apache.org/repos/asf/hive/blob/20908265/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/decimal_skewjoin.q.out 
b/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
new file mode 100644
index 000..a335aef
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
@@ -0,0 +1,77 @@
+PREHOOK: query: drop table if exists decimal_skewjoin_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_skewjoin_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists decimal_skewjoin_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_skewjoin_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_skewjoin_1 (t decimal(4,2), u decimal(5), 
v decimal)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_skewjoin_1
+POSTHOOK: query: create table decimal_skewjoin_1 (t decimal(4,2), u 
decimal(5), v decimal)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_skewjoin_1
+PREHOOK: query: create table decimal_skewjoin_2 (t decimal(4,2), u decimal(5), 
v decimal)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_skewjoin_2
+POSTHOOK: query: create table decimal_skewjoin_2 (t decimal(4,2), u

hive git commit: HIVE-14436: Hive 1.2.1/Hitting "ql.Driver: FAILED: IllegalArgumentException Error: , expected at the end of 'decimal(9'" after enabling hive.optimize.skewjoin and with MR engine (Dani

2016-08-09 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/master fc15086bd -> 63ada6d94


HIVE-14436: Hive 1.2.1/Hitting "ql.Driver: FAILED: IllegalArgumentException 
Error: , expected at the end of 'decimal(9'" after enabling 
hive.optimize.skewjoin and with MR engine (Daniel Dai, reviewed by Ashutosh 
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/63ada6d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/63ada6d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/63ada6d9

Branch: refs/heads/master
Commit: 63ada6d94a0617681cb715ef1f4ffbe5a77f9077
Parents: fc15086
Author: Daniel Dai 
Authored: Tue Aug 9 22:55:52 2016 -0700
Committer: Daniel Dai 
Committed: Tue Aug 9 22:55:52 2016 -0700

--
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 10 +--
 .../queries/clientpositive/decimal_skewjoin.q   | 21 ++
 .../clientpositive/decimal_skewjoin.q.out   | 77 
 3 files changed, 101 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/63ada6d9/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index fd25978..e78c3d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1878,13 +1878,9 @@ public final class Utilities {
   public static List getColumnTypes(Properties props) {
 List names = new ArrayList();
 String colNames = props.getProperty(serdeConstants.LIST_COLUMN_TYPES);
-String[] cols = colNames.trim().split(",");
-if (cols != null) {
-  for (String col : cols) {
-if (col != null && !col.trim().equals("")) {
-  names.add(col);
-}
-  }
+ArrayList cols = 
TypeInfoUtils.getTypeInfosFromTypeString(colNames);
+for (TypeInfo col : cols) {
+  names.add(col.getTypeName());
 }
 return names;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/63ada6d9/ql/src/test/queries/clientpositive/decimal_skewjoin.q
--
diff --git a/ql/src/test/queries/clientpositive/decimal_skewjoin.q 
b/ql/src/test/queries/clientpositive/decimal_skewjoin.q
new file mode 100644
index 000..cac0d89
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_skewjoin.q
@@ -0,0 +1,21 @@
+
+set hive.execution.engine=mr;
+set hive.optimize.skewjoin=true;
+set hive.auto.convert.join=false;
+
+drop table if exists decimal_skewjoin_1;
+drop table if exists decimal_skewjoin_2;
+
+create table decimal_skewjoin_1 (t decimal(4,2), u decimal(5), v decimal);
+create table decimal_skewjoin_2 (t decimal(4,2), u decimal(5), v decimal);
+
+insert overwrite table decimal_skewjoin_1
+  select cast('17.29' as decimal(4,2)), 3.1415926BD, 3115926.54321BD from src 
tablesample (1 rows);
+
+insert overwrite table decimal_skewjoin_2
+  select cast('17.29' as decimal(4,2)), 3.1415926BD, 3115926.54321BD from src 
tablesample (1 rows);
+
+select a.u from decimal_skewjoin_1 a INNER JOIN decimal_skewjoin_2 b ON 
a.t=b.t;
+
+drop table decimal_skewjoin_1;
+drop table decimal_skewjoin_2;

http://git-wip-us.apache.org/repos/asf/hive/blob/63ada6d9/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/decimal_skewjoin.q.out 
b/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
new file mode 100644
index 000..a335aef
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_skewjoin.q.out
@@ -0,0 +1,77 @@
+PREHOOK: query: drop table if exists decimal_skewjoin_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_skewjoin_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists decimal_skewjoin_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_skewjoin_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_skewjoin_1 (t decimal(4,2), u decimal(5), 
v decimal)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_skewjoin_1
+POSTHOOK: query: create table decimal_skewjoin_1 (t decimal(4,2), u 
decimal(5), v decimal)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_skewjoin_1
+PREHOOK: query: create table decimal_skewjoin_2 (t decimal(4,2), u decimal(5), 
v decimal)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_skewjoin_2
+POSTHOOK: query: create table decimal_skewjoin_2 (t decimal(4,2), u

hive git commit: HIVE-14428 : HadoopMetrics2Reporter leaks memory if the metrics sink is not configured correctly (Thejas Nair reviewed by Siddharth Seth)

2016-08-09 Thread thejas 
Repository: hive
Updated Branches:
  refs/heads/master f1d355372 -> fc15086bd


HIVE-14428 : HadoopMetrics2Reporter leaks memory if the metrics sink is not 
configured correctly (Thejas Nair reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fc15086b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fc15086b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fc15086b

Branch: refs/heads/master
Commit: fc15086bdb36305e72c3f3ff5d4aed99533b358b
Parents: f1d3553
Author: Thejas Nair 
Authored: Tue Aug 9 18:43:13 2016 -0700
Committer: Thejas Nair 
Committed: Tue Aug 9 18:43:42 2016 -0700

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fc15086b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 542df02..1365f91 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,7 +129,7 @@
 1.4
 10.10.2.0
 3.1.0
-
0.1.1
+
0.1.2
 14.0.1
 2.4.4
 2.6.1

hive git commit: HIVE-14428 : HadoopMetrics2Reporter leaks memory if the metrics sink is not configured correctly (Thejas Nair reviewed by Siddharth Seth)

2016-08-09 Thread thejas 
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 6b27b6a64 -> bc19ac1b2


HIVE-14428 : HadoopMetrics2Reporter leaks memory if the metrics sink is not 
configured correctly (Thejas Nair reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bc19ac1b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bc19ac1b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bc19ac1b

Branch: refs/heads/branch-2.1
Commit: bc19ac1b22709d0c486766a6a216ffb1f890f5f8
Parents: 6b27b6a
Author: Thejas Nair 
Authored: Tue Aug 9 18:43:13 2016 -0700
Committer: Thejas Nair 
Committed: Tue Aug 9 18:43:13 2016 -0700

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bc19ac1b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9c146ff..a4453ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,7 +129,7 @@
 1.4
 10.10.2.0
 3.1.0
-
0.1.1
+
0.1.2
 14.0.1
 2.4.4
 2.6.0

hive git commit: HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

2016-08-09 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 39a0e5509 -> 6b27b6a64


HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is 
disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

(cherry picked from commit f1d3553728d772b1db68d34066bd66864c2e66e5)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b27b6a6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b27b6a6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b27b6a6

Branch: refs/heads/branch-2.1
Commit: 6b27b6a64d65632e8369d75b4a53213d72a99186
Parents: 39a0e55
Author: Siddharth Seth 
Authored: Tue Aug 9 15:40:21 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Aug 9 15:41:15 2016 -0700

--
 .../hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java   | 3 +++
 .../hive/llap/tezplugins/TestLlapTaskSchedulerService.java  | 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6b27b6a6/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 6fa3107..efd774d 100644
--- 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -870,6 +870,9 @@ public class LlapTaskSchedulerService extends TaskScheduler 
{
 if (metrics != null) {
   metrics.setDisabledNodeCount(disabledNodesQueue.size());
 }
+// Trigger a scheduling run - in case there's some task which was 
waiting for this node to
+// become available.
+trySchedulingPendingTasks();
   }
 } finally {
   writeLock.unlock();

http://git-wip-us.apache.org/repos/asf/hive/blob/6b27b6a6/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
index afbab95..402658b 100644
--- 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
+++ 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -773,7 +773,7 @@ public class TestLlapTaskSchedulerService {
   // At this point. one p=2 task and task3(p=1) running. Ask for another 
p1 task.
   while (true) {
 tsWrapper.signalSchedulerRun();
-tsWrapper.awaitSchedulerRun();
+tsWrapper.awaitSchedulerRun(1000l);
 if (tsWrapper.ts.dagStats.numPreemptedTasks == 2) {
   break;
 }
@@ -1174,6 +1174,9 @@ public class TestLlapTaskSchedulerService {
   assertEquals(task2, argumentCaptor.getAllValues().get(1));
 
   reset(tsWrapper.mockAppCallback);
+  // Flush any pending scheduler runs which may be blocked. Wait 2 seconds 
for the run to complete.
+  tsWrapper.signalSchedulerRun();
+  tsWrapper.awaitSchedulerRun(2000l);
 
   // Mark a task as failed due to a comm failure.
   tsWrapper.deallocateTask(task1, false, 
TaskAttemptEndReason.COMMUNICATION_ERROR);

hive git commit: HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)

2016-08-09 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/master 2ef3ab855 -> f1d355372


HIVE-14439. LlapTaskScheduler should try scheduling tasks when a node is 
disabled. (Siddharth Seth, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f1d35537
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f1d35537
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f1d35537

Branch: refs/heads/master
Commit: f1d3553728d772b1db68d34066bd66864c2e66e5
Parents: 2ef3ab8
Author: Siddharth Seth 
Authored: Tue Aug 9 15:40:21 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Aug 9 15:40:21 2016 -0700

--
 .../hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java   | 3 +++
 .../hive/llap/tezplugins/TestLlapTaskSchedulerService.java  | 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f1d35537/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 6fa3107..efd774d 100644
--- 
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ 
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -870,6 +870,9 @@ public class LlapTaskSchedulerService extends TaskScheduler 
{
 if (metrics != null) {
   metrics.setDisabledNodeCount(disabledNodesQueue.size());
 }
+// Trigger a scheduling run - in case there's some task which was 
waiting for this node to
+// become available.
+trySchedulingPendingTasks();
   }
 } finally {
   writeLock.unlock();

http://git-wip-us.apache.org/repos/asf/hive/blob/f1d35537/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
--
diff --git 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
index afbab95..402658b 100644
--- 
a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
+++ 
b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -773,7 +773,7 @@ public class TestLlapTaskSchedulerService {
   // At this point. one p=2 task and task3(p=1) running. Ask for another 
p1 task.
   while (true) {
 tsWrapper.signalSchedulerRun();
-tsWrapper.awaitSchedulerRun();
+tsWrapper.awaitSchedulerRun(1000l);
 if (tsWrapper.ts.dagStats.numPreemptedTasks == 2) {
   break;
 }
@@ -1174,6 +1174,9 @@ public class TestLlapTaskSchedulerService {
   assertEquals(task2, argumentCaptor.getAllValues().get(1));
 
   reset(tsWrapper.mockAppCallback);
+  // Flush any pending scheduler runs which may be blocked. Wait 2 seconds 
for the run to complete.
+  tsWrapper.signalSchedulerRun();
+  tsWrapper.awaitSchedulerRun(2000l);
 
   // Mark a task as failed due to a comm failure.
   tsWrapper.deallocateTask(task1, false, 
TaskAttemptEndReason.COMMUNICATION_ERROR);

hive git commit: HIVE-7239 Fix bug in HiveIndexedInputFormat implementation that causes incorrect query result when input backed by Sequence/RC files (Illya Yalovyy via Ashutosh Chauhan)

2016-08-09 Thread hashutosh 
Repository: hive
Updated Branches:
  refs/heads/master ca807c63c -> 2ef3ab855


HIVE-7239 Fix bug in HiveIndexedInputFormat implementation that causes 
incorrect query result when input backed by Sequence/RC files (Illya Yalovyy 
via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ef3ab85
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ef3ab85
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ef3ab85

Branch: refs/heads/master
Commit: 2ef3ab855c725829cabf2c93b9fc9e31553ca643
Parents: ca807c6
Author: Illya Yalovyy 
Authored: Tue Jul 26 22:09:00 2016 -0800
Committer: Ashutosh Chauhan 
Committed: Tue Aug 9 12:43:39 2016 -0700

--
 .../hadoop/hive/ql/index/HiveIndexResult.java   |   3 +-
 .../hive/ql/index/HiveIndexedInputFormat.java   |  43 +--
 .../hadoop/hive/ql/index/IndexResult.java   |  25 ++
 .../hadoop/hive/ql/index/SplitFilter.java   | 125 
 .../hive/ql/index/MockHiveInputSplits.java  |  37 +++
 .../hadoop/hive/ql/index/MockIndexResult.java   |  38 +++
 .../hadoop/hive/ql/index/MockInputFile.java | 130 
 .../hive/ql/index/SplitFilterTestCase.java  | 153 ++
 .../ql/index/TestHiveInputSplitComparator.java  |  64 
 .../hadoop/hive/ql/index/TestSplitFilter.java   | 296 +++
 10 files changed, 874 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2ef3ab85/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java 
b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
index 33cc5c3..586cd68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.mapred.LineRecordReader.LineReader;
  * HiveIndexResult parses the input stream from an index query
  * to generate a list of file splits to query.
  */
-public class HiveIndexResult {
+public class HiveIndexResult implements IndexResult {
 
   public static final Logger l4j =
 LoggerFactory.getLogger(HiveIndexResult.class.getSimpleName());
@@ -182,6 +182,7 @@ public class HiveIndexResult {
 bucket.getOffsets().add(Long.parseLong(one_offset));
   }
 
+  @Override
   public boolean contains(FileSplit split) throws HiveException {
 
 if (buckets == null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/2ef3ab85/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
index 5247ece..0e6ec84 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.Arrays;
+import java.util.Collection;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,7 +37,6 @@ import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.io.IOPrepareCache;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputFormat;
@@ -144,49 +144,14 @@ public class HiveIndexedInputFormat extends 
HiveInputFormat {
 
 HiveInputSplit[] splits = (HiveInputSplit[]) this.doGetSplits(job, 
numSplits);
 
-ArrayList newSplits = new ArrayList(
-numSplits);
-
 long maxInputSize = HiveConf.getLongVar(job, 
ConfVars.HIVE_INDEX_COMPACT_QUERY_MAX_SIZE);
 if (maxInputSize < 0) {
   maxInputSize=Long.MAX_VALUE;
 }
 
-long sumSplitLengths = 0;
-for (HiveInputSplit split : splits) {
-  l4j.info("split start : " + split.getStart());
-  l4j.info("split end : " + (split.getStart() + split.getLength()));
+SplitFilter filter = new SplitFilter(hiveIndexResult, maxInputSize);
+Collection newSplits = filter.filter(splits);
 
-  try {
-if (hiveIndexResult.contains(split)) {
-  // we may miss a sync here
-  HiveInputSplit newSplit = split;
-  if (split.inputFormatClassName().contains("RCFile")
-  || split.inputFormatClassName().contains("SequenceFile")) {
-if

hive git commit: HIVE-14399: Fix test flakiness of org.apache.hive.hcatalog.listener.TestDbNotificationListener.cleanupNotifs (Daniel Dai, reviewed by Ashutosh Chauhan)

2016-08-09 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/master 3d4408f7d -> ca807c63c


HIVE-14399: Fix test flakiness of 
org.apache.hive.hcatalog.listener.TestDbNotificationListener.cleanupNotifs 
(Daniel Dai, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ca807c63
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ca807c63
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ca807c63

Branch: refs/heads/master
Commit: ca807c63ce6b026a368c074ffa1aacdc41322733
Parents: 3d4408f
Author: Daniel Dai 
Authored: Mon Aug 8 23:30:00 2016 -0700
Committer: Daniel Dai 
Committed: Mon Aug 8 23:30:00 2016 -0700

--
 .../hcatalog/listener/DbNotificationListener.java   |  5 +++--
 .../listener/TestDbNotificationListener.java| 16 
 2 files changed, 19 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ca807c63/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
--
diff --git 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 172f58d..0b3d891 100644
--- 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -265,7 +265,7 @@ public class DbNotificationListener extends 
MetaStoreEventListener {
   private static class CleanerThread extends Thread {
 private RawStore rs;
 private int ttl;
-
+static private long sleepTime = 6;
 
 CleanerThread(HiveConf conf, RawStore rs) {
   super("CleanerThread");
@@ -281,8 +281,9 @@ public class DbNotificationListener extends 
MetaStoreEventListener {
 synchronized(NOTIFICATION_TBL_LOCK) {
   rs.cleanNotificationEvents(ttl);
 }
+LOG.debug("Cleaner thread done");
 try {
-  Thread.sleep(6);
+  Thread.sleep(sleepTime);
 } catch (InterruptedException e) {
   LOG.info("Cleaner thread sleep interupted", e);
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/ca807c63/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index d0802d6..81ce67b 100644
--- 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -48,6 +48,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -58,12 +59,14 @@ public class TestDbNotificationListener {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(TestDbNotificationListener.class.getName());
   private static final int EVENTS_TTL = 30;
+  private static final int CLEANUP_SLEEP_TIME = 10;
   private static Map emptyParameters = new HashMap();
   private static IMetaStoreClient msClient;
   private static Driver driver;
   private int startTime;
   private long firstEventId;
 
+  @SuppressWarnings("rawtypes")
   @BeforeClass
   public static void connectToMetastore() throws Exception {
 HiveConf conf = new HiveConf();
@@ -73,6 +76,16 @@ public class TestDbNotificationListener {
 conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
 conf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true);
 conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+Class dbNotificationListener =
+
Class.forName("org.apache.hive.hcatalog.listener.DbNotificationListener");
+Class[] classes = dbNotificationListener.getDeclaredClasses();
+for (Class c : classes) {
+  if (c.getName().endsWith("CleanerThread")) {
+Field sleepTimeField = c.getDeclaredField("sleepTime");
+sleepTimeField.setAccessible(true);
+sleepTimeField.set(null, CLEANUP_SLEEP_TIME * 1000);
+  }
+}
 conf
 .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,

hive git commit: HIVE-14399: Fix test flakiness of org.apache.hive.hcatalog.listener.TestDbNotificationListener.cleanupNotifs (Daniel Dai, reviewed by Ashutosh Chauhan)

2016-08-09 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 759a965ce -> 39a0e5509


HIVE-14399: Fix test flakiness of 
org.apache.hive.hcatalog.listener.TestDbNotificationListener.cleanupNotifs 
(Daniel Dai, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/39a0e550
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/39a0e550
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/39a0e550

Branch: refs/heads/branch-2.1
Commit: 39a0e5509a11eae0ba89f092bf75948c971cc7cf
Parents: 759a965
Author: Daniel Dai 
Authored: Mon Aug 8 23:20:58 2016 -0700
Committer: Daniel Dai 
Committed: Mon Aug 8 23:20:58 2016 -0700

--
 .../hcatalog/listener/DbNotificationListener.java   |  5 +++--
 .../listener/TestDbNotificationListener.java| 16 
 2 files changed, 19 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/39a0e550/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
--
diff --git 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 8a49bf7..7245b9e 100644
--- 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -261,7 +261,7 @@ public class DbNotificationListener extends 
MetaStoreEventListener {
   private static class CleanerThread extends Thread {
 private RawStore rs;
 private int ttl;
-
+static private long sleepTime = 6;
 
 CleanerThread(HiveConf conf, RawStore rs) {
   super("CleanerThread");
@@ -275,8 +275,9 @@ public class DbNotificationListener extends 
MetaStoreEventListener {
 public void run() {
   while (true) {
 rs.cleanNotificationEvents(ttl);
+LOG.debug("Cleaner thread done");
 try {
-  Thread.sleep(6);
+  Thread.sleep(sleepTime);
 } catch (InterruptedException e) {
   LOG.info("Cleaner thread sleep interupted", e);
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/39a0e550/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index d0802d6..81ce67b 100644
--- 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -48,6 +48,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -58,12 +59,14 @@ public class TestDbNotificationListener {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(TestDbNotificationListener.class.getName());
   private static final int EVENTS_TTL = 30;
+  private static final int CLEANUP_SLEEP_TIME = 10;
   private static Map emptyParameters = new HashMap();
   private static IMetaStoreClient msClient;
   private static Driver driver;
   private int startTime;
   private long firstEventId;
 
+  @SuppressWarnings("rawtypes")
   @BeforeClass
   public static void connectToMetastore() throws Exception {
 HiveConf conf = new HiveConf();
@@ -73,6 +76,16 @@ public class TestDbNotificationListener {
 conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
 conf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true);
 conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+Class dbNotificationListener =
+
Class.forName("org.apache.hive.hcatalog.listener.DbNotificationListener");
+Class[] classes = dbNotificationListener.getDeclaredClasses();
+for (Class c : classes) {
+  if (c.getName().endsWith("CleanerThread")) {
+Field sleepTimeField = c.getDeclaredField("sleepTime");
+sleepTimeField.setAccessible(true);
+sleepTimeField.set(null, CLEANUP_SLEEP_TIME * 1000);
+  }
+}
 conf
 .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,