[02/46] hive git commit: HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE (Prasanth Jayachandran reviewed by Gopal V)

2018-07-03 Thread sershe 
HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE 
(Prasanth Jayachandran reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/221dbe08
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/221dbe08
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/221dbe08

Branch: refs/heads/master-txnstats
Commit: 221dbe085950e198d0766d60d6b00a70b30e5935
Parents: b122aea
Author: Prasanth Jayachandran 
Authored: Sun Jul 1 23:42:48 2018 -0700
Committer: Prasanth Jayachandran 
Committed: Sun Jul 1 23:42:48 2018 -0700

--
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  4 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 41 
 2 files changed, 44 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/221dbe08/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 21f8268..949a9e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -224,7 +224,9 @@ public class FileSinkOperator extends 
TerminalOperator implements
 private void commit(FileSystem fs, List commitPaths) throws 
HiveException {
   for (int idx = 0; idx < outPaths.length; ++idx) {
 try {
-  commitOneOutPath(idx, fs, commitPaths);
+  if (outPaths[idx] != null) {
+commitOneOutPath(idx, fs, commitPaths);
+  }
 } catch (IOException e) {
   throw new HiveException("Unable to commit output from: " +
   outPaths[idx] + " to: " + finalPaths[idx], e);

http://git-wip-us.apache.org/repos/asf/hive/blob/221dbe08/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index cf68d32..bbe9d5a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -184,6 +184,47 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
   }
 
+  @Test
+  public void testNoBucketsDP() throws Exception {
+hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+int[][] sourceVals1 = {{0,0,0},{3,3,3}};
+int[][] sourceVals2 = {{1,1,1},{2,2,2}};
+int[][] sourceVals3 = {{3,3,3},{4,4,4}};
+int[][] sourceVals4 = {{5,5,5},{6,6,6}};
+runStatementOnDriver("drop table if exists tmp");
+runStatementOnDriver("create table tmp (c1 integer, c2 integer) 
partitioned by (c3 integer) stored as orc " +
+  "tblproperties('transactional'='false')");
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals3));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals4));
+runStatementOnDriver("drop table if exists nobuckets");
+runStatementOnDriver("create table nobuckets (c1 integer, c2 integer) 
partitioned by (c3 integer) stored " +
+  "as orc tblproperties('transactional'='true', 
'transactional_properties'='default')");
+String stmt = "insert into nobuckets partition(c3) select * from tmp";
+runStatementOnDriver(stmt);
+List rs = runStatementOnDriver(
+  "select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by 
ROW__ID");
+Assert.assertEquals("", 8, rs.size());
+LOG.warn("after insert");
+for(String s : rs) {
+  LOG.warn(s);
+}
+
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2 in (0,3)");
+Assert.assertEquals(3, rs.size());
+runStatementOnDriver("update nobuckets set c2 = 17 where c2 in(0,3)");
+rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME 
from nobuckets order by INPUT__FILE__NAME, ROW__ID");
+LOG.warn("after update");
+for(String s : rs) {
+  LOG.warn(s);
+}
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2=17");
+Assert.assertEquals(3, rs.size());
+  }
+
   /**
* See CTAS tests in TestAcidOnTez
*/

hive git commit: HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE (Prasanth Jayachandran reviewed by Gopal V)

2018-07-02 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/master b122aea4e -> 221dbe085


HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE 
(Prasanth Jayachandran reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/221dbe08
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/221dbe08
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/221dbe08

Branch: refs/heads/master
Commit: 221dbe085950e198d0766d60d6b00a70b30e5935
Parents: b122aea
Author: Prasanth Jayachandran 
Authored: Sun Jul 1 23:42:48 2018 -0700
Committer: Prasanth Jayachandran 
Committed: Sun Jul 1 23:42:48 2018 -0700

--
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  4 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 41 
 2 files changed, 44 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/221dbe08/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 21f8268..949a9e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -224,7 +224,9 @@ public class FileSinkOperator extends 
TerminalOperator implements
 private void commit(FileSystem fs, List commitPaths) throws 
HiveException {
   for (int idx = 0; idx < outPaths.length; ++idx) {
 try {
-  commitOneOutPath(idx, fs, commitPaths);
+  if (outPaths[idx] != null) {
+commitOneOutPath(idx, fs, commitPaths);
+  }
 } catch (IOException e) {
   throw new HiveException("Unable to commit output from: " +
   outPaths[idx] + " to: " + finalPaths[idx], e);

http://git-wip-us.apache.org/repos/asf/hive/blob/221dbe08/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index cf68d32..bbe9d5a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -184,6 +184,47 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
   }
 
+  @Test
+  public void testNoBucketsDP() throws Exception {
+hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+int[][] sourceVals1 = {{0,0,0},{3,3,3}};
+int[][] sourceVals2 = {{1,1,1},{2,2,2}};
+int[][] sourceVals3 = {{3,3,3},{4,4,4}};
+int[][] sourceVals4 = {{5,5,5},{6,6,6}};
+runStatementOnDriver("drop table if exists tmp");
+runStatementOnDriver("create table tmp (c1 integer, c2 integer) 
partitioned by (c3 integer) stored as orc " +
+  "tblproperties('transactional'='false')");
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals3));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals4));
+runStatementOnDriver("drop table if exists nobuckets");
+runStatementOnDriver("create table nobuckets (c1 integer, c2 integer) 
partitioned by (c3 integer) stored " +
+  "as orc tblproperties('transactional'='true', 
'transactional_properties'='default')");
+String stmt = "insert into nobuckets partition(c3) select * from tmp";
+runStatementOnDriver(stmt);
+List rs = runStatementOnDriver(
+  "select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by 
ROW__ID");
+Assert.assertEquals("", 8, rs.size());
+LOG.warn("after insert");
+for(String s : rs) {
+  LOG.warn(s);
+}
+
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2 in (0,3)");
+Assert.assertEquals(3, rs.size());
+runStatementOnDriver("update nobuckets set c2 = 17 where c2 in(0,3)");
+rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME 
from nobuckets order by INPUT__FILE__NAME, ROW__ID");
+LOG.warn("after update");
+for(String s : rs) {
+  LOG.warn(s);
+}
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2=17");
+Assert.assertEquals(3, rs.size());
+  }
+
   /**
* See CTAS tests in TestAcidOnTez
*/

hive git commit: HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE (Prasanth Jayachandran reviewed by Gopal V)

2018-07-02 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/branch-3 62a382419 -> 4edeefd60


HIVE-20038: Update queries on non-bucketed + partitioned tables throws NPE 
(Prasanth Jayachandran reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4edeefd6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4edeefd6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4edeefd6

Branch: refs/heads/branch-3
Commit: 4edeefd60a79814d39c40bd2702820008ddfd488
Parents: 62a3824
Author: Prasanth Jayachandran 
Authored: Sun Jul 1 23:42:48 2018 -0700
Committer: Prasanth Jayachandran 
Committed: Sun Jul 1 23:46:33 2018 -0700

--
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  4 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 41 
 2 files changed, 44 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/4edeefd6/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 2a74f86..56c32bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -232,7 +232,9 @@ public class FileSinkOperator extends 
TerminalOperator implements
 private void commit(FileSystem fs, List commitPaths) throws 
HiveException {
   for (int idx = 0; idx < outPaths.length; ++idx) {
 try {
-  commitOneOutPath(idx, fs, commitPaths);
+  if (outPaths[idx] != null) {
+commitOneOutPath(idx, fs, commitPaths);
+  }
 } catch (IOException e) {
   throw new HiveException("Unable to commit output from: " +
   outPaths[idx] + " to: " + finalPaths[idx], e);

http://git-wip-us.apache.org/repos/asf/hive/blob/4edeefd6/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index cf68d32..bbe9d5a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -184,6 +184,47 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
   }
 
+  @Test
+  public void testNoBucketsDP() throws Exception {
+hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+int[][] sourceVals1 = {{0,0,0},{3,3,3}};
+int[][] sourceVals2 = {{1,1,1},{2,2,2}};
+int[][] sourceVals3 = {{3,3,3},{4,4,4}};
+int[][] sourceVals4 = {{5,5,5},{6,6,6}};
+runStatementOnDriver("drop table if exists tmp");
+runStatementOnDriver("create table tmp (c1 integer, c2 integer) 
partitioned by (c3 integer) stored as orc " +
+  "tblproperties('transactional'='false')");
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals3));
+runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals4));
+runStatementOnDriver("drop table if exists nobuckets");
+runStatementOnDriver("create table nobuckets (c1 integer, c2 integer) 
partitioned by (c3 integer) stored " +
+  "as orc tblproperties('transactional'='true', 
'transactional_properties'='default')");
+String stmt = "insert into nobuckets partition(c3) select * from tmp";
+runStatementOnDriver(stmt);
+List rs = runStatementOnDriver(
+  "select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by 
ROW__ID");
+Assert.assertEquals("", 8, rs.size());
+LOG.warn("after insert");
+for(String s : rs) {
+  LOG.warn(s);
+}
+
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2 in (0,3)");
+Assert.assertEquals(3, rs.size());
+runStatementOnDriver("update nobuckets set c2 = 17 where c2 in(0,3)");
+rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME 
from nobuckets order by INPUT__FILE__NAME, ROW__ID");
+LOG.warn("after update");
+for(String s : rs) {
+  LOG.warn(s);
+}
+rs = runStatementOnDriver(
+  "select * from nobuckets where c2=17");
+Assert.assertEquals(3, rs.size());
+  }
+
   /**
* See CTAS tests in TestAcidOnTez
*/