date:20181116

hive git commit: HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna Guruvayoor Murali)

2018-11-16 Thread vihangk1

Repository: hive
Updated Branches:
  refs/heads/branch-3 4663e50e7 -> ec89f32a3


HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter 
database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna 
Guruvayoor Murali)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec89f32a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec89f32a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec89f32a

Branch: refs/heads/branch-3
Commit: ec89f32a31f1623c97012829618f1ad79b741bd7
Parents: 4663e50
Author: Vihang Karajgaonkar 
Authored: Fri Nov 16 10:59:41 2018 -0800
Committer: Vihang Karajgaonkar 
Committed: Fri Nov 16 14:31:53 2018 -0800

--
 .../listener/TestDbNotificationListener.java| 35 
 .../json/JSONCreateDatabaseMessage.java |  2 +-
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ec89f32a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 82429e3..06f5d5c 100644
--- 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.hive.metastore.events.ListenerEvent;
 import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent;
 import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
 import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage;
 import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
 import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
 import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
@@ -320,6 +321,7 @@ public class TestDbNotificationListener {
 // Parse the message field
 CreateDatabaseMessage createDbMsg = 
md.getCreateDatabaseMessage(event.getMessage());
 assertEquals(dbName, createDbMsg.getDB());
+assertEquals(db, createDbMsg.getDatabaseObject());
 
 // Verify the eventID was passed to the non-transactional listener
 
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, 
firstEventId + 1);
@@ -339,6 +341,39 @@ public class TestDbNotificationListener {
   }
 
   @Test
+  public void alterDatabase() throws Exception {
+String dbName = "alterdb";
+String dbLocationUri = "file:/tmp";
+String dbDescription = "no description";
+msClient.createDatabase(new Database(dbName, dbDescription, dbLocationUri, 
emptyParameters));
+// get the db for comparison below since it may include additional 
parameters
+Database dbBefore = msClient.getDatabase(dbName);
+// create alter database notification
+String newDesc = "test database";
+Database dbAfter = dbBefore.deepCopy();
+dbAfter.setDescription(newDesc);
+msClient.alterDatabase(dbName, dbAfter);
+dbAfter = msClient.getDatabase(dbName);
+
+// Read notification from metastore
+NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 
0, null);
+assertEquals(2, rsp.getEventsSize());
+// check the contents of alter database notification
+NotificationEvent event = rsp.getEvents().get(1);
+assertEquals(firstEventId + 2, event.getEventId());
+assertTrue(event.getEventTime() >= startTime);
+assertEquals(EventType.ALTER_DATABASE.toString(), event.getEventType());
+assertEquals(dbName, event.getDbName());
+assertNull(event.getTableName());
+
+// Parse the message field
+AlterDatabaseMessage alterDatabaseMessage = 
md.getAlterDatabaseMessage(event.getMessage());
+assertEquals(dbName, alterDatabaseMessage.getDB());
+assertEquals(dbBefore, alterDatabaseMessage.getDbObjBefore());
+assertEquals(dbAfter, alterDatabaseMessage.getDbObjAfter());
+  }
+
+  @Test
   public void dropDatabase() throws Exception {
 String dbName = "dropdb";
 String dbName2 = "dropdb2";

http://git-wip-us.apache.org/repos/asf/hive/blob/ec89f32a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/

hive git commit: HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna Guruvayoor Murali)

2018-11-16 Thread vihangk1

Repository: hive
Updated Branches:
  refs/heads/master c737dbd8f -> fd5f34fdd


HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter 
database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna 
Guruvayoor Murali)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fd5f34fd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fd5f34fd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fd5f34fd

Branch: refs/heads/master
Commit: fd5f34fdd9be45b05f7cd7c771b01ff9dd1765fc
Parents: c737dbd
Author: Vihang Karajgaonkar 
Authored: Fri Nov 16 10:59:41 2018 -0800
Committer: Vihang Karajgaonkar 
Committed: Fri Nov 16 10:59:58 2018 -0800

--
 .../listener/TestDbNotificationListener.java| 35 
 .../json/JSONCreateDatabaseMessage.java |  2 +-
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fd5f34fd/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 3e404df..d4b7b02 100644
--- 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.hive.metastore.events.ListenerEvent;
 import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent;
 import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
 import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage;
 import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
 import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
 import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
@@ -341,6 +342,7 @@ public class TestDbNotificationListener {
 // Parse the message field
 CreateDatabaseMessage createDbMsg = 
md.getCreateDatabaseMessage(event.getMessage());
 assertEquals(dbName, createDbMsg.getDB());
+assertEquals(db, createDbMsg.getDatabaseObject());
 
 // Verify the eventID was passed to the non-transactional listener
 
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, 
firstEventId + 1);
@@ -364,6 +366,39 @@ public class TestDbNotificationListener {
   }
 
   @Test
+  public void alterDatabase() throws Exception {
+String dbName = "alterdb";
+String dbLocationUri = testTempDir;
+String dbDescription = "no description";
+msClient.createDatabase(new Database(dbName, dbDescription, dbLocationUri, 
emptyParameters));
+// get the db for comparison below since it may include additional 
parameters
+Database dbBefore = msClient.getDatabase(dbName);
+// create alter database notification
+String newDesc = "test database";
+Database dbAfter = dbBefore.deepCopy();
+dbAfter.setDescription(newDesc);
+msClient.alterDatabase(dbName, dbAfter);
+dbAfter = msClient.getDatabase(dbName);
+
+// Read notification from metastore
+NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 
0, null);
+assertEquals(2, rsp.getEventsSize());
+// check the contents of alter database notification
+NotificationEvent event = rsp.getEvents().get(1);
+assertEquals(firstEventId + 2, event.getEventId());
+assertTrue(event.getEventTime() >= startTime);
+assertEquals(EventType.ALTER_DATABASE.toString(), event.getEventType());
+assertEquals(dbName, event.getDbName());
+assertNull(event.getTableName());
+
+// Parse the message field
+AlterDatabaseMessage alterDatabaseMessage = 
md.getAlterDatabaseMessage(event.getMessage());
+assertEquals(dbName, alterDatabaseMessage.getDB());
+assertEquals(dbBefore, alterDatabaseMessage.getDbObjBefore());
+assertEquals(dbAfter, alterDatabaseMessage.getDbObjAfter());
+  }
+
+  @Test
   public void dropDatabase() throws Exception {
 String dbName = "dropdb";
 String dbName2 = "dropdb2";

http://git-wip-us.apache.org/repos/asf/hive/blob/fd5f34fd/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
--
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
 
b/standalone-metastore/me

hive git commit: HIVE-20360: QTest: ignore driver/qtest exclusions if -Dqfile param is set (Laszlo Bodor via )

2018-11-16 Thread kgyrtkirk

Repository: hive
Updated Branches:
  refs/heads/master dd5121c7b -> c737dbd8f


HIVE-20360: QTest: ignore driver/qtest exclusions if -Dqfile param is set 
(Laszlo Bodor via )

Signed-off-by: Zoltan Haindrich 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c737dbd8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c737dbd8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c737dbd8

Branch: refs/heads/master
Commit: c737dbd8f1db679e20c04b2e58808ce3dbbd0710
Parents: dd5121c7
Author: Laszlo Bodor 
Authored: Fri Nov 16 13:06:24 2018 +0100
Committer: Zoltan Haindrich 
Committed: Fri Nov 16 13:06:24 2018 +0100

--
 .../hive/cli/control/AbstractCliConfig.java | 17 ++--
 .../apache/hadoop/hive/ql/QOutProcessor.java| 45 +---
 .../org/apache/hadoop/hive/ql/QTestUtil.java| 35 +--
 .../hadoop/hive/ql/TestQOutProcessor.java   |  2 +-
 4 files changed, 62 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
--
diff --git 
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
 
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index 7151372..142f903 100644
--- 
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ 
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.QTestUtil;
 import org.apache.hadoop.hive.ql.QTestUtil.FsType;
 import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
 import org.apache.hive.testutils.HiveTestEnvSetup;
@@ -214,7 +215,7 @@ public abstract class AbstractCliConfig {
 
 // dedup file list
 Set testFiles = new TreeSet<>();
-if (queryFile != null && !queryFile.equals("")) {
+if (isQFileSpecified()) {
   // The user may have passed a list of files - comma separated
   for (String qFile : TEST_SPLITTER.split(queryFile)) {
 File qF;
@@ -223,10 +224,10 @@ public abstract class AbstractCliConfig {
 } else {
   qF = new File(qFile);
 }
-if (excludedQueryFileNames.contains(qFile)) {
+if (excludedQueryFileNames.contains(qFile) && !isQFileSpecified()) {
   LOG.warn(qF.getAbsolutePath() + " is among the excluded query files 
for this driver."
   + " Please update CliConfigs.java or 
testconfiguration.properties file to"
-  + " include the qfile");
+  + " include the qfile or specify qfile through command line 
explicitly: -Dqfile=test.q");
 }
 testFiles.add(qF);
   }
@@ -241,12 +242,20 @@ public abstract class AbstractCliConfig {
 }
 
 for (String qFileName : excludedQueryFileNames) {
-  testFiles.remove(new File(queryDir, qFileName));
+  // in case of running as ptest, exclusions should be respected,
+  // because test drivers receive every qfiles regardless of exclusions
+  if ("hiveptest".equals(System.getProperty("user.name")) || 
!isQFileSpecified()) {
+testFiles.remove(new File(queryDir, qFileName));
+  }
 }
 
 return testFiles;
   }
 
+  public boolean isQFileSpecified() {
+return queryFile != null && !queryFile.equals("");
+  }
+
   private void prepareDirs() throws Exception {
 File hiveRootDir = new File(HIVE_ROOT);
 if (!hiveRootDir.exists()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
--
diff --git 
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java 
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index a17c508..254cc95 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -25,7 +25,9 @@ import java.io.FileOutputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -52,6 +54,10 @@ public class QOutProcessor {
   public static final String MASK_PATTERN = " A masked pattern was here 
";
   public static final String PARTIAL_MASK_PATTERN = " A PARTIAL masked 
pattern was here ";
 
+  private final Set qMaskStatsQuerySet = new HashSe

hive git commit: HIVE-14557: Nullpointer When both SkewJoin and Mapjoin Enabled (Nemon Lou reviewed by Yongzhi and Rui)

2018-11-16 Thread lirui

Repository: hive
Updated Branches:
  refs/heads/master 156ed46a3 -> dd5121c7b


HIVE-14557: Nullpointer When both SkewJoin and Mapjoin Enabled (Nemon Lou 
reviewed by Yongzhi and Rui)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dd5121c7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dd5121c7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dd5121c7

Branch: refs/heads/master
Commit: dd5121c7be9bac03e12dbd2db7e3b863cefe851e
Parents: 156ed46
Author: Rui Li 
Authored: Fri Nov 16 19:20:46 2018 +0800
Committer: Rui Li 
Committed: Fri Nov 16 19:20:46 2018 +0800

--
 .../optimizer/physical/PhysicalOptimizer.java   |   6 +-
 .../runtime_skewjoin_mapjoin_spark.q.out| 529 +++
 2 files changed, 77 insertions(+), 458 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/dd5121c7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
index d508d02..6c6908a 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
@@ -46,9 +46,6 @@ public class PhysicalOptimizer {
*/
   private void initialize(HiveConf hiveConf) {
 resolvers = new ArrayList();
-if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESKEWJOIN)) {
-  resolvers.add(new SkewJoinResolver());
-}
 if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) {
   resolvers.add(new CommonJoinResolver());
 
@@ -59,6 +56,9 @@ public class PhysicalOptimizer {
 resolvers.add(new SortMergeJoinResolver());
   }
 }
+if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESKEWJOIN)) {
+  resolvers.add(new SkewJoinResolver());
+}
 
 resolvers.add(new MapJoinResolver());
 if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/dd5121c7/ql/src/test/results/clientpositive/runtime_skewjoin_mapjoin_spark.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/runtime_skewjoin_mapjoin_spark.q.out 
b/ql/src/test/results/clientpositive/runtime_skewjoin_mapjoin_spark.q.out
index ee8dbc2..3ecf14f 100644
--- a/ql/src/test/results/clientpositive/runtime_skewjoin_mapjoin_spark.q.out
+++ b/ql/src/test/results/clientpositive/runtime_skewjoin_mapjoin_spark.q.out
@@ -35,39 +35,80 @@ POSTHOOK: Input: default@src
 POSTHOOK: Input: default@t1_n94
  A masked pattern was here 
 STAGE DEPENDENCIES:
-  Stage-18 is a root stage , consists of Stage-22, Stage-23, Stage-24, Stage-1
-  Stage-22 has a backup stage: Stage-1
-  Stage-15 depends on stages: Stage-22
-  Stage-8 depends on stages: Stage-1, Stage-11, Stage-12, Stage-13, Stage-15, 
Stage-16, Stage-17 , consists of Stage-20, Stage-21, Stage-2
-  Stage-20
-  Stage-6 depends on stages: Stage-20
-  Stage-2 depends on stages: Stage-6, Stage-7
-  Stage-21
-  Stage-7 depends on stages: Stage-21
-  Stage-23 has a backup stage: Stage-1
-  Stage-16 depends on stages: Stage-23
-  Stage-24 has a backup stage: Stage-1
-  Stage-17 depends on stages: Stage-24
+  Stage-14 is a root stage
+  Stage-10 depends on stages: Stage-14
+  Stage-9 depends on stages: Stage-10 , consists of Stage-11, Stage-12, 
Stage-13, Stage-1
+  Stage-11 has a backup stage: Stage-1
+  Stage-6 depends on stages: Stage-11
+  Stage-2 depends on stages: Stage-1, Stage-6, Stage-7, Stage-8
+  Stage-12 has a backup stage: Stage-1
+  Stage-7 depends on stages: Stage-12
+  Stage-13 has a backup stage: Stage-1
+  Stage-8 depends on stages: Stage-13
   Stage-1
-  Stage-29 is a root stage
-  Stage-19 depends on stages: Stage-29
-  Stage-10 depends on stages: Stage-19 , consists of Stage-28, Stage-1
-  Stage-28
-  Stage-9 depends on stages: Stage-28
-  Stage-14 depends on stages: Stage-9 , consists of Stage-25, Stage-26, 
Stage-27, Stage-1
-  Stage-25 has a backup stage: Stage-1
-  Stage-11 depends on stages: Stage-25
-  Stage-26 has a backup stage: Stage-1
-  Stage-12 depends on stages: Stage-26
-  Stage-27 has a backup stage: Stage-1
-  Stage-13 depends on stages: Stage-27
   Stage-0 depends on stages: Stage-2
 
 STAGE PLANS:
-  Stage: Stage-18
+  Stage: Stage-14
+Map Reduce Local Work
+  Alias -> Map Local Tables:
+$hdt$_2:$hdt$_3:t1_n94 
+  Fetch Operator
+limit: -1
+  Alias -> Map Local Operator Tree:
+$hdt$_2:$hdt$_3:t1_n94 
+  TableScan
+alias: t1_n94
+filterExpr: key is not null (t

hive git commit: HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna Guruvayoor Murali)

hive git commit: HIVE-20916 : Fix typo in JSONCreateDatabaseMessage and add test for alter database (Vihang Karajgaonkar, reviewed by Andrew Sherman and Bharathkrishna Guruvayoor Murali)

hive git commit: HIVE-20360: QTest: ignore driver/qtest exclusions if -Dqfile param is set (Laszlo Bodor via )

hive git commit: HIVE-14557: Nullpointer When both SkewJoin and Mapjoin Enabled (Nemon Lou reviewed by Yongzhi and Rui)

4 matches

Site Navigation

Mail list logo

Footer information