[hive] branch master updated: HIVE-26685: Improve path name escaping/unescaping (#3721)

[weiz]

This is an automated email from the ASF dual-hosted git repository.

weiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new dbe2a323351 HIVE-26685: Improve path name escaping/unescaping (#3721)
dbe2a323351 is described below

commit dbe2a323351b7a0196fc7834023b9bc28cd3244e
Author: James Petty 
AuthorDate: Mon Dec 5 13:04:54 2022 -0500

HIVE-26685: Improve path name escaping/unescaping (#3721)
---
 .../org/apache/hadoop/hive/common/FileUtils.java   | 38 +++---
 .../apache/hadoop/hive/common/TestFileUtils.java   |  8 +
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java 
b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 37ff2c04dc2..17169d6e184 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -258,6 +258,11 @@ public final class FileUtils {
 }
   }
 
+  /**
+   * Hex encoding characters indexed by integer value
+   */
+  private static final char[] HEX_UPPER_CHARS = {'0', '1', '2', '3', '4', '5', 
'6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
   static boolean needsEscaping(char c) {
 return c < charToEscape.size() && charToEscape.get(c);
   }
@@ -287,12 +292,28 @@ public final class FileUtils {
   }
 }
 
-StringBuilder sb = new StringBuilder();
+//  Fast-path detection, no escaping and therefore no copying necessary
+int firstEscapeIndex = -1;
 for (int i = 0; i < path.length(); i++) {
+  if (needsEscaping(path.charAt(i))) {
+firstEscapeIndex = i;
+break;
+  }
+}
+if (firstEscapeIndex == -1) {
+  return path;
+}
+
+// slow path, escape beyond the first required escape character into a new 
string
+StringBuilder sb = new StringBuilder();
+if (firstEscapeIndex > 0) {
+  sb.append(path, 0, firstEscapeIndex);
+}
+
+for (int i = firstEscapeIndex; i < path.length(); i++) {
   char c = path.charAt(i);
   if (needsEscaping(c)) {
-sb.append('%');
-sb.append(String.format("%1$02X", (int) c));
+sb.append('%').append(HEX_UPPER_CHARS[(0xF0 & c) >>> 
4]).append(HEX_UPPER_CHARS[(0x0F & c)]);
   } else {
 sb.append(c);
   }
@@ -301,8 +322,17 @@ public final class FileUtils {
   }
 
   public static String unescapePathName(String path) {
+int firstUnescapeIndex = path.indexOf('%');
+if (firstUnescapeIndex == -1) {
+  return path;
+}
+
 StringBuilder sb = new StringBuilder();
-for (int i = 0; i < path.length(); i++) {
+if (firstUnescapeIndex > 0) {
+  sb.append(path, 0, firstUnescapeIndex);
+}
+
+for (int i = firstUnescapeIndex; i < path.length(); i++) {
   char c = path.charAt(i);
   if (c == '%' && i + 2 < path.length()) {
 int code = -1;
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java 
b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
index 2721deb7a03..9ffb52ba5f9 100644
--- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
@@ -303,6 +303,14 @@ public class TestFileUtils {
 assertEquals(1, assertExpectedFilePaths(itr, 
Collections.singletonList("mock:/tmp/dummy")));
   }
 
+  @Test
+  public void testPathEscapeChars() {
+StringBuilder sb = new StringBuilder();
+FileUtils.charToEscape.stream().forEach(integer -> sb.append((char) 
integer));
+String path = sb.toString();
+assertEquals(path, 
FileUtils.unescapePathName(FileUtils.escapePathName(path)));
+  }
+
   private int assertExpectedFilePaths(RemoteIterator 
lfs, List expectedPaths)
   throws Exception {
 int count = 0;

hive git commit: HIVE-20858 : Serializer is not correctly initialized with configuration in Utilities.createEmptyBuckets (Wei Zheng, reviewed by Daniel Dai)

[weiz]

Repository: hive
Updated Branches:
  refs/heads/master f4a48fdbb -> 6d713b656


HIVE-20858 : Serializer is not correctly initialized with configuration in 
Utilities.createEmptyBuckets (Wei Zheng, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6d713b65
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6d713b65
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6d713b65

Branch: refs/heads/master
Commit: 6d713b6564ecb9d1ae0db66c3742d2a8bc347211
Parents: f4a48fd
Author: Wei Zheng 
Authored: Tue Nov 6 13:27:53 2018 -0800
Committer: Wei Zheng 
Committed: Tue Nov 6 13:27:53 2018 -0800

--
 ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6d713b65/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 65d49cb..d0f6451 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1576,7 +1576,7 @@ public final class Utilities {
 Class outputClass = null;
 try {
   Serializer serializer = (Serializer) 
tableInfo.getDeserializerClass().newInstance();
-  serializer.initialize(null, tableInfo.getProperties());
+  serializer.initialize(hconf, tableInfo.getProperties());
   outputClass = serializer.getSerializedClass();
   hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf, 
tableInfo);
 } catch (SerDeException e) {

[3/7] hive git commit: HIVE-17419: ANALYZE TABLE...COMPUTE STATISTICS FOR COLUMNS command shows computed stats for masked tables (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[weiz]

HIVE-17419: ANALYZE TABLE...COMPUTE STATISTICS FOR COLUMNS command shows 
computed stats for masked tables (Jesus Camacho Rodriguez, reviewed by Ashutosh 
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/65480cec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/65480cec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/65480cec

Branch: refs/heads/hive-14535
Commit: 65480cecfd329d096bb701196eb57838fcfda218
Parents: 5f26f39
Author: Jesus Camacho Rodriguez 
Authored: Fri Sep 8 09:14:34 2017 -0700
Committer: Jesus Camacho Rodriguez 
Committed: Fri Sep 8 09:14:34 2017 -0700

--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  5 +++--
 ql/src/test/queries/clientpositive/masking_11.q |  6 ++
 .../results/clientpositive/masking_11.q.out | 20 
 3 files changed, 29 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/65480cec/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 6f379da..cebb0af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3513,7 +3513,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
   pos = Integer.valueOf(pos.intValue() + 1);
   matched++;
 
-  if (unparseTranslator.isEnabled() || tableMask.isEnabled()) {
+  if (unparseTranslator.isEnabled() || (tableMask.isEnabled() && 
analyzeRewrite == null)) {
 if (replacementText.length() > 0) {
   replacementText.append(", ");
 }
@@ -11331,7 +11331,8 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
 // 2. Gen OP Tree from resolved Parse Tree
 Operator sinkOp = genOPTree(ast, plannerCtx);
 
-if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) {
+if (!unparseTranslator.isEnabled() &&
+(tableMask.isEnabled() && analyzeRewrite == null)) {
   // Here we rewrite the * and also the masking table
   ASTNode tree = rewriteASTWithMaskAndFilter(ast);
   if (tree != ast) {

http://git-wip-us.apache.org/repos/asf/hive/blob/65480cec/ql/src/test/queries/clientpositive/masking_11.q
--
diff --git a/ql/src/test/queries/clientpositive/masking_11.q 
b/ql/src/test/queries/clientpositive/masking_11.q
new file mode 100644
index 000..dc46117
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/masking_11.q
@@ -0,0 +1,6 @@
+set hive.mapred.mode=nonstrict;
+set 
hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+create table `masking_test` as select cast(key as int) as key, value from src;
+
+analyze table `masking_test` compute statistics for columns;

http://git-wip-us.apache.org/repos/asf/hive/blob/65480cec/ql/src/test/results/clientpositive/masking_11.q.out
--
diff --git a/ql/src/test/results/clientpositive/masking_11.q.out 
b/ql/src/test/results/clientpositive/masking_11.q.out
new file mode 100644
index 000..f29c51f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/masking_11.q.out
@@ -0,0 +1,20 @@
+PREHOOK: query: create table `masking_test` as select cast(key as int) as key, 
value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create table `masking_test` as select cast(key as int) as 
key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, 
type:string, comment:default), ]
+POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, 
type:string, comment:default), ]
+PREHOOK: query: analyze table `masking_test` compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test
+ A masked pattern was here 
+POSTHOOK: query: analyze table `masking_test` compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test
+ A masked pattern was here 

[6/7] hive git commit: HIVE-17480: repl dump sub dir should use UUID instead of timestamp (Tao Li, reviewed by Daniel Dai)

[weiz]

HIVE-17480: repl dump sub dir should use UUID instead of timestamp (Tao Li, 
reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5df15407
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5df15407
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5df15407

Branch: refs/heads/hive-14535
Commit: 5df15407f0cc0ca3dc01f1258f270f49b6ca6e4c
Parents: a74107e
Author: Daniel Dai 
Authored: Fri Sep 8 14:31:56 2017 -0700
Committer: Daniel Dai 
Committed: Fri Sep 8 14:31:56 2017 -0700

--
 .../java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5df15407/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 7703f31..95eb2db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.ql.exec.repl;
 
 import com.google.common.primitives.Ints;
+
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,9 +51,9 @@ import 
org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandler;
 import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandlerFactory;
 import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
 import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
-import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 import org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger;
 import org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,6 +61,7 @@ import org.slf4j.LoggerFactory;
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 public class ReplDumpTask extends Task implements Serializable {
   private static final String dumpSchema = 
"dump_dir,last_repl_id#string,string";
@@ -274,7 +276,7 @@ public class ReplDumpTask extends Task 
implements Serializable {
 return ReplDumpWork.testInjectDumpDir;
   }
 } else {
-  return String.valueOf(System.currentTimeMillis());
+  return UUID.randomUUID().toString();
   // TODO: time good enough for now - we'll likely improve this.
   // We may also work in something the equivalent of pid, thrid and move 
to nanos to ensure
   // uniqueness.

[5/7] hive git commit: HIVE-17475: Disable mapjoin using hint (Deepak Jaiswal reviewed by Jason Dere)

[weiz]

HIVE-17475: Disable mapjoin using hint (Deepak Jaiswal reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a74107ef
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a74107ef
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a74107ef

Branch: refs/heads/hive-14535
Commit: a74107ef1dfacd88921dbac1c694641302af6044
Parents: 5e57a2a
Author: Jason Dere 
Authored: Fri Sep 8 14:03:27 2017 -0700
Committer: Jason Dere 
Committed: Fri Sep 8 14:03:27 2017 -0700

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   4 +-
 .../hadoop/hive/ql/parse/ParseContext.java  |  10 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  27 +
 .../test/queries/clientpositive/mapjoin_hint.q  |  69 ++
 .../clientpositive/llap/mapjoin_hint.q.out  | 643 +++
 6 files changed, 753 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a74107ef/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 663c95b..d9e760f 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -542,6 +542,7 @@ minillaplocal.query.files=\
   load_dyn_part5.q,\
   lvj_mapjoin.q,\
   mapjoin_decimal.q,\
+  mapjoin_hint.q,\
   mapjoin_emit_interval.q,\
   mergejoin_3way.q,\
   mrr.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/a74107ef/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index a2414f3..9412e5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -103,7 +103,9 @@ public class ConvertJoinMapJoin implements NodeProcessor {
 joinOp.getConf().setMemoryMonitorInfo(memoryMonitorInfo);
 
 TezBucketJoinProcCtx tezBucketJoinProcCtx = new 
TezBucketJoinProcCtx(context.conf);
-if (!context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) {
+boolean hiveConvertJoin = 
context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN) &
+!context.parseContext.getDisableMapJoin();
+if (!hiveConvertJoin) {
   // we are just converting to a common merge join operator. The shuffle
   // join in map-reduce case.
   Object retval = checkAndConvertSMBJoin(context, joinOp, 
tezBucketJoinProcCtx, maxSize);

http://git-wip-us.apache.org/repos/asf/hive/blob/a74107ef/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
index adf7f84..9ab42f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
@@ -133,6 +133,8 @@ public class ParseContext {
   new HashMap<>();
 
   private Map semiJoinHints;
+  private boolean disableMapJoin;
+
   public ParseContext() {
   }
 
@@ -705,4 +707,12 @@ public class ParseContext {
   public Map getSemiJoinHints() {
 return semiJoinHints;
   }
+
+  public void setDisableMapJoin(boolean disableMapJoin) {
+this.disableMapJoin = disableMapJoin;
+  }
+
+  public boolean getDisableMapJoin() {
+return disableMapJoin;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a74107ef/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index cebb0af..1c74779 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -9118,6 +9118,31 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
   }
 
   /**
+   * disableMapJoinWithHint
+   * @param hints
+   * @return true if hint to disable hint is provided, else false
+   * @throws SemanticException
+   */
+  private boolean disableMapJoinWithHint(List hints) throws 
SemanticException {
+if (hints == null || hints.size() == 0) return false;
+for (ASTNode 

[4/7] hive git commit: HIVE-17359 Deal with TypeInfo dependencies in the metastore. This closes #239. (Alan Gates, reviewed by Owen O'Malley)

[weiz]

HIVE-17359 Deal with TypeInfo dependencies in the metastore.  This closes #239. 
 (Alan Gates, reviewed by Owen O'Malley)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e57a2a8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e57a2a8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e57a2a8

Branch: refs/heads/hive-14535
Commit: 5e57a2a8675760659a7a6a48c8cd39c2b2bf5dc0
Parents: 65480ce
Author: Alan Gates 
Authored: Fri Sep 8 13:02:21 2017 -0700
Committer: Alan Gates 
Committed: Fri Sep 8 13:02:21 2017 -0700

--
 .../hadoop/hive/metastore/HiveAlterHandler.java |  24 +-
 .../hive/metastore/MetaStoreDirectSql.java  |   7 +-
 .../hadoop/hive/metastore/ObjectStore.java  |  11 +-
 .../metastore/PartitionExpressionProxy.java |  10 +-
 .../hive/metastore/cache/CachedStore.java   |  12 +-
 .../hive/metastore/parser/ExpressionTree.java   |   6 +-
 .../MockPartitionExpressionForMetastore.java|   5 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |  11 +-
 .../hadoop/hive/metastore/TestOldSchema.java|   6 +-
 .../ppr/PartitionExpressionForMetastore.java|  14 +-
 .../hadoop/hive/metastore/ColumnType.java   | 242 +++
 11 files changed, 297 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java 
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 28c3cfe..b279e1d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -156,8 +156,7 @@ public class HiveAlterHandler implements AlterHandler {
 false)) {
 // Throws InvalidOperationException if the new column types are not
 // compatible with the current column types.
-MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(
-oldt.getSd().getCols(), newt.getSd().getCols());
+checkColTypeChangeCompatible(oldt.getSd().getCols(), 
newt.getSd().getCols());
   }
 
   //check that partition keys have not changed, except for virtual views
@@ -852,4 +851,25 @@ public class HiveAlterHandler implements AlterHandler {
 
 return newPartsColStats;
   }
+
+  private void checkColTypeChangeCompatible(List oldCols, 
List newCols)
+  throws InvalidOperationException {
+List incompatibleCols = new ArrayList<>();
+int maxCols = Math.min(oldCols.size(), newCols.size());
+for (int i = 0; i < maxCols; i++) {
+  if (!ColumnType.areColTypesCompatible(
+  ColumnType.getTypeName(oldCols.get(i).getType()),
+  ColumnType.getTypeName(newCols.get(i).getType( {
+incompatibleCols.add(newCols.get(i).getName());
+  }
+}
+if (!incompatibleCols.isEmpty()) {
+  throw new InvalidOperationException(
+  "The following columns have types incompatible with the existing " +
+  "columns in their respective positions :\n" +
+  org.apache.commons.lang.StringUtils.join(incompatibleCols, ',')
+  );
+}
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java 
b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 2dd7b68..b122c43 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -76,7 +76,6 @@ import 
org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
-import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hive.common.util.BloomFilter;
 import org.datanucleus.store.rdbms.query.ForwardQueryResult;
 import org.slf4j.Logger;
@@ -1112,11 +,11 @@ class MetaStoreDirectSql {
   Invalid;
 
   static FilterType fromType(String colTypeStr) {
-if (colTypeStr.equals(serdeConstants.STRING_TYPE_NAME)) {
+if (colTypeStr.equals(ColumnType.STRING_TYPE_NAME)) {
   return FilterType.String;
-} else if (colTypeStr.equals(serdeConstants.DATE_TYPE_NAME)) {
+

[2/7] hive git commit: HIVE-17468: Shade and package appropriate jackson version for druid storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez)

[weiz]

HIVE-17468: Shade and package appropriate jackson version for druid storage 
handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5f26f393
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5f26f393
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5f26f393

Branch: refs/heads/hive-14535
Commit: 5f26f393ecc0cca4afd3a2ed3c1d46acf47701d1
Parents: 09afd83
Author: Slim Bouguerra 
Authored: Fri Sep 8 09:08:29 2017 -0700
Committer: Jesus Camacho Rodriguez 
Committed: Fri Sep 8 09:08:29 2017 -0700

--
 druid-handler/pom.xml   | 48 +---
 .../hive/druid/DruidStorageHandlerUtils.java|  7 +++
 .../hadoop/hive/druid/io/DruidOutputFormat.java |  9 ++--
 .../druid/io/DruidQueryBasedInputFormat.java|  8 +---
 .../hadoop/hive/druid/io/DruidRecordWriter.java |  3 +-
 .../serde/DruidGroupByQueryRecordReader.java|  5 +-
 .../serde/DruidSelectQueryRecordReader.java |  5 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java | 15 +++---
 .../serde/DruidTimeseriesQueryRecordReader.java |  5 +-
 .../druid/serde/DruidTopNQueryRecordReader.java |  5 +-
 .../TestHiveDruidQueryBasedInputFormat.java |  6 +--
 .../hive/ql/io/TestDruidRecordWriter.java   | 19 +---
 12 files changed, 66 insertions(+), 69 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5f26f393/druid-handler/pom.xml
--
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
index 81c744f..48b2af9 100644
--- a/druid-handler/pom.xml
+++ b/druid-handler/pom.xml
@@ -53,6 +53,18 @@
   com.google.guava
   guava
 
+
+  com.fasterxml.jackson.core
+  jackson-core
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
   
 
 
@@ -119,18 +131,6 @@
   ${druid.version}
   
 
-  com.fasterxml.jackson.core
-  jackson-core
-
-
-  com.fasterxml.jackson.core
-  jackson-annotations
-
-
-  com.fasterxml.jackson.core
-  jackson-databind
-
-
   
   com.google.code.findbugs
   annotations
@@ -216,24 +216,20 @@
   com.google.guava
   guava
 
-  
-
-
-  org.apache.calcite
-  calcite-druid
-  ${calcite.version}
-  
 
-  org.apache.calcite.avatica
-  avatica-core
+  com.fasterxml.jackson.core
+  jackson-core
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
 
   
 
-
-  org.apache.calcite.avatica
-  avatica
-  ${avatica.version}
-
 
 
   junit

http://git-wip-us.apache.org/repos/asf/hive/blob/5f26f393/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
--
diff --git 
a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
 
b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
index 3eeb0c3..7169140 100644
--- 
a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
+++ 
b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
@@ -41,6 +41,7 @@ import io.druid.timeline.partition.NumberedShardSpec;
 import io.druid.timeline.partition.PartitionChunk;
 import io.druid.timeline.partition.ShardSpec;
 
+import org.apache.calcite.adapter.druid.LocalInterval;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -77,6 +78,7 @@ import org.jboss.netty.handler.codec.http.HttpHeaders;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
+import org.joda.time.chrono.ISOChronology;
 import org.joda.time.format.ISODateTimeFormat;
 import org.skife.jdbi.v2.FoldController;
 import org.skife.jdbi.v2.Folder3;
@@ -122,7 +124,12 @@ public final class DruidStorageHandlerUtils {
   private static final Logger LOG = 
LoggerFactory.getLogger(DruidStorageHandlerUtils.class);
 
   private static final String SMILE_CONTENT_TYPE = 
"application/x-jackson-smile";
+  public static final String DEFAULT_TIMESTAMP_COLUMN = "__time";
 
+  public static final Interval DEFAULT_INTERVAL = new Interval(
+  new DateTime("1900-01-01", 

[1/7] hive git commit: HIVE-17450: rename TestTxnCommandsBase (Peter Vary reviewed by Zoltan Haindrich)

[weiz]

Repository: hive
Updated Branches:
  refs/heads/hive-14535 00e751007 -> bb1c0f25f


HIVE-17450: rename TestTxnCommandsBase (Peter Vary reviewed by Zoltan Haindrich)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09afd83d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09afd83d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09afd83d

Branch: refs/heads/hive-14535
Commit: 09afd83dd0cacebcd85171318582090fff0f0fe4
Parents: 8482c5f
Author: Peter Vary 
Authored: Fri Sep 8 11:37:58 2017 +0200
Committer: Peter Vary 
Committed: Fri Sep 8 11:37:58 2017 +0200

--
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  11 +-
 .../hadoop/hive/ql/TestTxnCommandsBase.java | 162 ---
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |   2 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 162 +++
 4 files changed, 164 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/09afd83d/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 0f129fc..4f1c7d8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hive.ql;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
@@ -34,27 +33,19 @@ import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.BucketCodec;
-import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
 import org.junit.After;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Ignore;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileOutputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
@@ -69,7 +60,7 @@ import java.util.concurrent.TimeUnit;
  * Tests here are for multi-statement transactions (WIP) and those that don't 
need to
  * run with Acid 2.0 (see subclasses of TestTxnCommands2)
  */
-public class TestTxnCommands extends TestTxnCommandsBase {
+public class TestTxnCommands extends TxnCommandsBaseForTests {
   static final private Logger LOG = 
LoggerFactory.getLogger(TestTxnCommands.class);
   private static final String TEST_DATA_DIR = new 
File(System.getProperty("java.io.tmpdir") +
 File.separator + TestTxnCommands.class.getCanonicalName()

http://git-wip-us.apache.org/repos/asf/hive/blob/09afd83d/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
deleted file mode 100644
index d6e709d..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
+++ /dev/null
@@ -1,162 +0,0 @@
-package org.apache.hadoop.hive.ql;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.io.HiveInputFormat;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public abstract class TestTxnCommandsBase {
-  //bucket count for test tables; set it to 1 for easier debugging
-  final static int 

[7/7] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

HIVE-14671 : merge master into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bb1c0f25
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bb1c0f25
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bb1c0f25

Branch: refs/heads/hive-14535
Commit: bb1c0f25fc17d8c5ced8855a05b6651358c5fd96
Parents: 00e7510 5df1540
Author: Wei Zheng 
Authored: Fri Sep 8 15:35:21 2017 -0700
Committer: Wei Zheng 
Committed: Fri Sep 8 15:35:21 2017 -0700

--
 druid-handler/pom.xml   |  48 +-
 .../hive/druid/DruidStorageHandlerUtils.java|   7 +
 .../hadoop/hive/druid/io/DruidOutputFormat.java |   9 +-
 .../druid/io/DruidQueryBasedInputFormat.java|   8 +-
 .../hadoop/hive/druid/io/DruidRecordWriter.java |   3 +-
 .../serde/DruidGroupByQueryRecordReader.java|   5 +-
 .../serde/DruidSelectQueryRecordReader.java |   5 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java |  15 +-
 .../serde/DruidTimeseriesQueryRecordReader.java |   5 +-
 .../druid/serde/DruidTopNQueryRecordReader.java |   5 +-
 .../TestHiveDruidQueryBasedInputFormat.java |   6 +-
 .../hive/ql/io/TestDruidRecordWriter.java   |  19 +-
 .../test/resources/testconfiguration.properties |   1 +
 .../hadoop/hive/metastore/HiveAlterHandler.java |  24 +-
 .../hive/metastore/MetaStoreDirectSql.java  |   7 +-
 .../hadoop/hive/metastore/ObjectStore.java  |  11 +-
 .../metastore/PartitionExpressionProxy.java |  10 +-
 .../hive/metastore/cache/CachedStore.java   |  12 +-
 .../hive/metastore/parser/ExpressionTree.java   |   6 +-
 .../MockPartitionExpressionForMetastore.java|   5 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |  11 +-
 .../hadoop/hive/metastore/TestOldSchema.java|   6 +-
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   6 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   4 +-
 .../ppr/PartitionExpressionForMetastore.java|  14 +-
 .../hadoop/hive/ql/parse/ParseContext.java  |  10 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  32 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  11 +-
 .../hadoop/hive/ql/TestTxnCommandsBase.java | 162 -
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |   2 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 162 +
 .../test/queries/clientpositive/mapjoin_hint.q  |  69 ++
 ql/src/test/queries/clientpositive/masking_11.q |   6 +
 .../clientpositive/llap/mapjoin_hint.q.out  | 643 +++
 .../results/clientpositive/masking_11.q.out |  20 +
 .../hadoop/hive/metastore/ColumnType.java   | 242 +++
 36 files changed, 1313 insertions(+), 298 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/itests/src/test/resources/testconfiguration.properties
--

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
--

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
--

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
--
diff --cc 
metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 7573fb2,5448d0d..75eba79
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -290,11 -272,11 +287,11 @@@ public class TestObjectStore 
public void testPartitionOps() throws MetaException, 
InvalidObjectException, NoSuchObjectException, InvalidInputException {
  Database db1 = new Database(DB1, "description", "locationurl", null);
  objectStore.createDatabase(db1);
 -StorageDescriptor sd = new StorageDescriptor(null, "location", null, 
null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, 
null, null);
 +StorageDescriptor sd = createFakeSd("location");
  HashMap tableParams = new HashMap();
  tableParams.put("EXTERNAL", "false");
- FieldSchema partitionKey1 = new FieldSchema("Country", 
serdeConstants.STRING_TYPE_NAME, "");
- FieldSchema partitionKey2 = new FieldSchema("State", 
serdeConstants.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey1 = new FieldSchema("Country", 
ColumnType.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey2 = new FieldSchema("State", 
ColumnType.STRING_TYPE_NAME, "");
  Table tbl1 = 

hive git commit: HIVE-14990 : run all tests for MM tables and fix the issues that are found (Wei Zheng)

[weiz]

Repository: hive
Updated Branches:
  refs/heads/hive-14535 18d133857 -> 00e751007


HIVE-14990 : run all tests for MM tables and fix the issues that are found (Wei 
Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/00e75100
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/00e75100
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/00e75100

Branch: refs/heads/hive-14535
Commit: 00e75100725cb3972555481c51e11e10bbcefbdf
Parents: 18d1338
Author: Wei Zheng 
Authored: Fri Sep 8 15:35:05 2017 -0700
Committer: Wei Zheng 
Committed: Fri Sep 8 15:35:05 2017 -0700

--
 ql/src/test/results/clientpositive/mm_all.q.out | 35 ++--
 1 file changed, 33 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/00e75100/ql/src/test/results/clientpositive/mm_all.q.out
--
diff --git a/ql/src/test/results/clientpositive/mm_all.q.out 
b/ql/src/test/results/clientpositive/mm_all.q.out
index b69ac00..ea60414 100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@ -1637,21 +1637,34 @@ POSTHOOK: Input: default@intermediate@p=455
 POSTHOOK: Input: default@intermediate@p=456
 POSTHOOK: Input: default@intermediate@p=457
 POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=455
+POSTHOOK: Output: default@multi1_mm@p=456
+POSTHOOK: Output: default@multi1_mm@p=457
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE 
[(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE 
[(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE 
[(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE 
[(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE 
[(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE 
[(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE 
[(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE 
[(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE 
[(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE 
[(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 PREHOOK: type: QUERY
 PREHOOK: Input: default@multi1_mm
 PREHOOK: Input: default@multi1_mm@p=1
 PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
  A masked pattern was here 
 POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi1_mm
 POSTHOOK: Input: default@multi1_mm@p=1
 POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: default@multi1_mm@p=456
+POSTHOOK: Input: default@multi1_mm@p=457
  A masked pattern was here 
 0  456 1
 0  456 1
@@ -1673,16 +1686,22 @@ POSTHOOK: Input: default@multi1_mm@p=2
 103457 2
 45597  1
 45597  2
+45597  455
 45598  1
 45598  2
+45598  455
 4560   1
 4560   2
+4560   456
 45610  1
 45610  2
+45610  456
 457100 1
 457100 2
+457100 457
 457103 1
 457103 2
+457103 457
 PREHOOK: query: from intermediate
 insert into table multi1_mm partition(p) select p, key, 1
 insert into table multi1_mm partition(p=1) select key, p
@@ -1711,12 +1730,18 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@multi1_mm
 PREHOOK: Input: default@multi1_mm@p=1
 PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
  A masked pattern was here 
 POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi1_mm
 POSTHOOK: Input: default@multi1_mm@p=1
 POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: 

[57/74] [abbrv] hive git commit: HIVE-17426 : Execution framework in hive to run tasks in parallel other than MR Tasks (Anishek Agarwal, reviewed by Sankar Hariappan, Thejas Nair)

[weiz]

HIVE-17426 : Execution framework in hive to run tasks in parallel other than MR 
Tasks (Anishek Agarwal, reviewed by Sankar Hariappan, Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb0034c0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb0034c0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb0034c0

Branch: refs/heads/hive-14535
Commit: eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a
Parents: bff8d95
Author: Thejas M Nair 
Authored: Tue Sep 5 00:07:01 2017 -0700
Committer: Thejas M Nair 
Committed: Tue Sep 5 00:07:11 2017 -0700

--
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  | 18 ++
 .../hive/ql/parse/BaseSemanticAnalyzer.java | 26 +---
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  7 +++---
 .../dump/BootStrapReplicationSpecFunction.java  |  1 -
 .../hive/ql/parse/repl/dump/HiveWrapper.java|  5 
 5 files changed, 32 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/eb0034c0/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 3ebd3cc..165a2e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
 import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 import org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger;
 import org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
-import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -239,12 +239,15 @@ public class ReplDumpTask extends Task 
implements Serializable {
   private void dumpTable(String dbName, String tblName, Path dbRoot) throws 
Exception {
 try {
   Hive db = getHive();
-  TableSpec ts = new TableSpec(db, conf, dbName + "." + tblName, null);
+  HiveWrapper.Tuple tuple = new HiveWrapper(db, 
dbName).table(tblName);
+  TableSpec tableSpec = new TableSpec(tuple.object);
   TableExport.Paths exportPaths =
   new TableExport.Paths(work.astRepresentationForErrorMsg, dbRoot, 
tblName, conf);
   String distCpDoAsUser = 
conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
-  new TableExport(exportPaths, ts, getNewReplicationSpec(), db, 
distCpDoAsUser, conf).write();
-  replLogger.tableLog(tblName, ts.tableHandle.getTableType());
+  tuple.replicationSpec.setIsReplace(true);  // by default for all other 
objects this is false
+  new TableExport(exportPaths, tableSpec, tuple.replicationSpec, db, 
distCpDoAsUser, conf).write();
+
+  replLogger.tableLog(tblName, tableSpec.tableHandle.getTableType());
 } catch (InvalidTableException te) {
   // Bootstrap dump shouldn't fail if the table is dropped/renamed while 
dumping it.
   // Just log a debug message and skip it.
@@ -252,13 +255,6 @@ public class ReplDumpTask extends Task 
implements Serializable {
 }
   }
 
-  private ReplicationSpec getNewReplicationSpec() throws TException {
-ReplicationSpec rspec = getNewReplicationSpec("replv2", "will-be-set");
-rspec.setCurrentReplicationState(String.valueOf(getHive().getMSC()
-.getCurrentNotificationEventId().getEventId()));
-return rspec;
-  }
-
   private ReplicationSpec getNewReplicationSpec(String evState, String 
objState) {
 return new ReplicationSpec(true, false, evState, objState, false, true, 
true);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/eb0034c0/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index 136e951..3ad30c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -24,6 +24,7 @@ import 

[08/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out 
b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index 4f0795e..ea3c78b 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -5590,14 +5590,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t1
-  Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: c1 (type: int)
 outputColumnNames: _col0
-Statistics: Num rows: 4 Data size: 313 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 4 Data size: 313 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: int)
 Execution mode: llap
 LLAP IO: no inputs
@@ -5605,30 +5605,30 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t2
-  Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: c1 (type: int)
 outputColumnNames: c1
-Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 3 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count(), count(c1)
   mode: hash
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint), _col1 (type: 
bigint)
   Group By Operator
 keys: c1 (type: int)
 mode: hash
 outputColumnNames: _col0
-Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 3 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 3 Data size: 6 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 3 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -5641,12 +5641,12 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 4 Data size: 381 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 4 Data size: 100 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 4 Data size: 381 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 4 Data size: 100 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col1 (type: bigint), _col2 (type: bigint)
 Reducer 3 
 Execution mode: llap
@@ -5658,17 +5658,17 @@ STAGE PLANS:
   0 _col0 (type: int)
   1 _col0 (type: int)
 outputColumnNames: _col0, _col1, _col2, _col4
-Statistics: Num rows: 4 Data size: 419 Basic stats: COMPLETE 
Column stats: NONE
+

[23/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/mergejoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out 
b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
index 9df5e42..d1c9c57 100644
--- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
@@ -293,19 +293,19 @@ STAGE PLANS:
 TableScan
   alias: a
   filterExpr: key is not null (type: boolean)
-  Statistics: Num rows: 242 Data size: 24684 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 23672 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 242 Data size: 24684 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 22498 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 242 Data size: 24684 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 230 Data size: 22498 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 242 Data size: 24684 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 22498 Basic 
stats: COMPLETE Column stats: NONE
 Execution mode: vectorized, llap
 LLAP IO: all inputs
 Map 4 
@@ -313,19 +313,19 @@ STAGE PLANS:
 TableScan
   alias: b
   filterExpr: key is not null (type: boolean)
-  Statistics: Num rows: 500 Data size: 51000 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 48904 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 51000 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 46458 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 500 Data size: 51000 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 475 Data size: 46458 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 500 Data size: 51000 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 46458 Basic 
stats: COMPLETE Column stats: NONE
 Execution mode: vectorized, llap
 LLAP IO: all inputs
 Reducer 2 
@@ -337,7 +337,7 @@ STAGE PLANS:
 keys:
   0 _col0 (type: int)
   1 _col0 (type: int)
-Statistics: Num rows: 550 Data size: 56100 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 522 Data size: 51103 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   mode: hash
@@ -1382,32 +1382,32 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 242 Data size: 24684 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 23672 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: int)
 outputColumnNames: _col0
-Statistics: Num rows: 242 Data size: 24684 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 242 Data size: 23672 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 242 Data size: 24684 Basic stats: 

[67/74] [abbrv] hive git commit: HIVE-17152 : Improve security of random generator for HS2 cookies (Tao Li via Thejas Nair)

[weiz]

HIVE-17152 : Improve security of random generator for HS2 cookies (Tao Li via 
Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bb4035b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bb4035b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bb4035b6

Branch: refs/heads/hive-14535
Commit: bb4035b68f3e98e158466848ec0fd501f89137b6
Parents: 849fa02
Author: Tao LI 
Authored: Thu Sep 7 10:53:58 2017 -0700
Committer: Thejas M Nair 
Committed: Thu Sep 7 10:53:58 2017 -0700

--
 service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java | 4 ++--
 .../org/apache/hive/service/cli/thrift/ThriftHttpServlet.java| 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bb4035b6/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
--
diff --git a/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java 
b/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
index f11c0e4a..8b5661a 100644
--- a/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
+++ b/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
@@ -21,11 +21,11 @@ package org.apache.hive.service.auth;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.StringTokenizer;
 
@@ -96,7 +96,7 @@ public final class HttpAuthUtils {
 
sb.append(COOKIE_CLIENT_USER_NAME).append(COOKIE_KEY_VALUE_SEPARATOR).append(clientUserName).
 append(COOKIE_ATTR_SEPARATOR);
 sb.append(COOKIE_CLIENT_RAND_NUMBER).append(COOKIE_KEY_VALUE_SEPARATOR).
-append((new Random(System.currentTimeMillis())).nextLong());
+append((new SecureRandom()).nextLong());
 return sb.toString();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/bb4035b6/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
--
diff --git 
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java 
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index cda736c..f3bbf8a 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -22,11 +22,11 @@ import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -84,7 +84,7 @@ public class ThriftHttpServlet extends TServlet {
   // Class members for cookie based authentication.
   private CookieSigner signer;
   public static final String AUTH_COOKIE = "hive.server2.auth";
-  private static final Random RAN = new Random();
+  private static final SecureRandom RAN = new SecureRandom();
   private boolean isCookieAuthEnabled;
   private String cookieDomain;
   private String cookiePath;

[43/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out 
b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
index 649afb5..f763566 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
@@ -125,64 +125,71 @@ STAGE PLANS:
 Tez
  A masked pattern was here 
   Edges:
-Map 2 <- Map 1 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 45994 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 43713 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 230 Data size: 43713 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 43713 Basic 
stats: COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
-Map 2 
+Map 3 
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 94800 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 90060 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
-  Map Join Operator
-condition map:
- Inner Join 0 to 1
-keys:
-  0 _col0 (type: int)
-  1 _col0 (type: int)
-outputColumnNames: _col0, _col1, _col3
-input vertices:
-  0 Map 1
-Statistics: Num rows: 550 Data size: 10243 Basic 
stats: COMPLETE Column stats: NONE
-Select Operator
-  expressions: _col0 (type: int), _col1 (type: 
string), _col3 (type: string)
-  outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 550 Data size: 10243 Basic 
stats: COMPLETE Column stats: NONE
-  File Output Operator
-compressed: false
-Statistics: Num rows: 550 Data size: 10243 Basic 
stats: COMPLETE Column stats: NONE
-table:
-input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-serde: 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+  Statistics: Num rows: 475 Data size: 90060 Basic stats: 
COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col0 (type: int)
+sort order: +
+Map-reduce partition columns: _col0 (type: int)
+Statistics: Num rows: 475 Data 

[19/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out 
b/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
index bd3d247..b8cf9cb 100644
--- a/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
+++ b/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
@@ -58,59 +58,59 @@ STAGE PLANS:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: f1
-  Statistics: Num rows: 10 Data size: 170 Basic stats: 
COMPLETE Column stats: NONE
+  alias: p1
+  Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
   Filter Operator
-predicate: i is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 170 Basic stats: 
COMPLETE Column stats: NONE
+predicate: p_partkey is not null (type: boolean)
+Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
 Select Operator
-  expressions: i (type: int)
-  outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 170 Basic stats: 
COMPLETE Column stats: NONE
+  expressions: p_partkey (type: int), p_name (type: 
string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), 
p_size (type: int), p_container (type: string), p_retailprice (type: double), 
p_comment (type: string)
+  outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8
+  Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 10 Data size: 170 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
+value expressions: _col1 (type: string), _col2 (type: 
string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 
(type: string), _col7 (type: double), _col8 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
 Map 4 
 Map Operator Tree:
 TableScan
-  alias: p1
-  Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
+  alias: f1
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
-predicate: p_partkey is not null (type: boolean)
-Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
+predicate: i is not null (type: boolean)
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
-  expressions: p_partkey (type: int), p_name (type: 
string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), 
p_size (type: int), p_container (type: string), p_retailprice (type: double), 
p_comment (type: string)
-  outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8
-  Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
+  expressions: i (type: int)
+  outputColumnNames: _col0
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 26 Data size: 16094 Basic stats: 
COMPLETE Column stats: COMPLETE
-value expressions: _col1 (type: string), _col2 (type: 
string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 
(type: string), _col7 (type: double), _col8 (type: string)
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 5 
 Map Operator Tree:
 TableScan
   alias: f2
-  

[32/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out 
b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
index 453711c..fe86a60 100644
--- 
a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
+++ 
b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
@@ -173,19 +173,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: over1k_orc
-  Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1049 Data size: 23952 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (t is null or (t = 27)) (type: boolean)
-Statistics: Num rows: 1048 Data size: 310873 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: si (type: smallint), i (type: int), b 
(type: bigint), f (type: float), t (type: tinyint)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4
-  Statistics: Num rows: 1048 Data size: 310873 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col4 (type: tinyint), _col0 (type: 
smallint)
 sort order: ++
 Map-reduce partition columns: _col4 (type: tinyint)
-Statistics: Num rows: 1048 Data size: 310873 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: int), _col2 (type: 
bigint), _col3 (type: float)
 Execution mode: vectorized, llap
 LLAP IO: all inputs
@@ -195,11 +195,11 @@ STAGE PLANS:
   Select Operator
 expressions: KEY._col0 (type: smallint), VALUE._col1 (type: 
int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: 
tinyint)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-Statistics: Num rows: 1048 Data size: 310873 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 57 Data size: 1301 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
   Dp Sort State: PARTITION_SORTED
-  Statistics: Num rows: 1048 Data size: 310873 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -248,20 +248,20 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: over1k_orc
-  Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1049 Data size: 23952 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (t is null or (t = 27)) (type: boolean)
-Statistics: Num rows: 1048 Data size: 310873 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: si (type: smallint), i (type: int), b 
(type: bigint), f (type: float), t (type: tinyint)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4
-  Statistics: Num rows: 1048 Data size: 310873 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 57 Data size: 1301 Basic stats: 
COMPLETE Column stats: NONE
   Limit
 Number of rows: 10
-Statistics: Num rows: 10 Data size: 2960 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 220 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 10 Data size: 2960 Basic 
stats: COMPLETE Column stats: NONE
+

[05/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/tez_smb_1.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_1.q.out 
b/ql/src/test/results/clientpositive/llap/tez_smb_1.q.out
index 2fcf0fc..a486b13 100644
--- a/ql/src/test/results/clientpositive/llap/tez_smb_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_smb_1.q.out
@@ -130,32 +130,32 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: s3
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 3490 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
 Map Operator Tree:
 TableScan
   alias: s1
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 3490 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
   Merge Join Operator
 condition map:
  Inner Join 0 to 1
 keys:
   0 _col0 (type: int)
   1 _col0 (type: int)
-Statistics: Num rows: 266 Data size: 4952 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 253 Data size: 3647 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   mode: hash
@@ -222,37 +222,37 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: vt1
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 242 Data size: 3490 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 242 Data size: 4502 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 230 Data size: 3316 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 4 
 Map Operator Tree:
 TableScan
   alias: t2
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 94800 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 

[03/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out 
b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
index 8d97fa4..c353280 100644
--- a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
@@ -5684,11 +5684,11 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: src_thrift
-  Statistics: Num rows: 11 Data size: 3070 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 11 Data size: 2024 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: astring (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 11 Data size: 3070 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 2024 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   keys: _col0 (type: string)
   mode: hash
@@ -6466,35 +6466,35 @@ STAGE PLANS:
  A masked pattern was here 
   Edges:
 Map 1 <- Union 2 (CONTAINS)
-Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS)
 Reducer 3 <- Union 2 (SIMPLE_EDGE)
+Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 2 
(CONTAINS)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
   alias: dst_union22_delta
-  Statistics: Num rows: 500 Data size: 20936 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 454856 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: (UDFToDouble(k0) <= 50.0) (type: boolean)
-Statistics: Num rows: 166 Data size: 6950 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 166 Data size: 151012 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: k1 (type: string), k2 (type: string), k3 
(type: string), k4 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3
-  Statistics: Num rows: 166 Data size: 6950 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 166 Data size: 151012 Basic stats: 
COMPLETE Column stats: NONE
   Group By Operator
 keys: _col0 (type: string), _col1 (type: string), 
_col2 (type: string), _col3 (type: string)
 mode: hash
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 348 Data size: 40548 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 348 Data size: 248973 Basic 
stats: COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: string), _col1 (type: 
string), _col2 (type: string), _col3 (type: string)
   null sort order: 
   sort order: 
   Map-reduce partition columns: _col0 (type: string), 
_col1 (type: string), _col2 (type: string), _col3 (type: string)
-  Statistics: Num rows: 348 Data size: 40548 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 348 Data size: 248973 Basic 
stats: COMPLETE Column stats: NONE
   tag: -1
   auto parallelism: true
 Execution mode: llap
@@ -6555,50 +6555,25 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 500 Data size: 103124 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 500 Data size: 278292 Basic stats: 
COMPLETE Column stats: PARTIAL
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: (UDFToDouble(k1) > 20.0) (type: boolean)
-Statistics: Num rows: 166 Data size: 30544 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 166 Data size: 89056 Basic stats: 
COMPLETE Column stats: PARTIAL
 Select Operator
   expressions: k1 (type: string), k2 (type: string), ds 
(type: string)
   

[47/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out 
b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out
index 7e3fe4c..37d97d2 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out
@@ -118,22 +118,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 1 Data size: 122 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 298 Basic stats: COMPLETE 
Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 122 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 298 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 122 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 298 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 null sort order: a
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 1 Data size: 122 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 298 Basic stats: 
COMPLETE Column stats: NONE
 tag: 0
 auto parallelism: true
 Execution mode: llap
@@ -195,16 +195,16 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 116 Data size: 12552 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 116 Data size: 32232 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 116 Data size: 12552 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 111 Data size: 30842 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 116 Data size: 12552 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 111 Data size: 30842 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -215,7 +215,7 @@ STAGE PLANS:
 input vertices:
   0 Map 1
 Position of Big Table: 1
-Statistics: Num rows: 127 Data size: 13807 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 122 Data size: 33926 Basic 
stats: COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   mode: hash
@@ -408,22 +408,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 1 Data size: 122 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 298 Basic stats: COMPLETE 
Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 122 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 298 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 122 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 298 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
  

[11/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out 
b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out
index 1589329..3c672c3 100644
--- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out
@@ -68,19 +68,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -88,19 +88,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -108,19 +108,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: c
-  Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic 

[29/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out 
b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
index 4400aae..4693c2b 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
@@ -1727,34 +1727,36 @@ Stage-0
 Stage-1
   Reducer 2 llap
   File Output Operator [FS_16]
-Merge Join Operator [MERGEJOIN_27] (rows=292 width=10)
-  Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"]
-<-Map 1 [SIMPLE_EDGE] llap
-  SHUFFLE [RS_12]
-PartitionCols:_col1
-Merge Join Operator [MERGEJOIN_25] (rows=266 width=10)
-  Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"]
-<-Select Operator [SEL_5] (rows=242 width=10)
-Output:["_col0"]
-Filter Operator [FIL_23] (rows=242 width=10)
-  predicate:key is not null
-  TableScan [TS_3] (rows=242 width=10)
-default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"]
-<-Select Operator [SEL_2] (rows=242 width=10)
-Output:["_col0","_col1"]
-Filter Operator [FIL_22] (rows=242 width=10)
-  predicate:(key is not null and value is not null)
-  TableScan [TS_0] (rows=242 width=10)
-default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
-<-Map 4 [SIMPLE_EDGE] llap
-  SHUFFLE [RS_13]
-PartitionCols:_col1
-Select Operator [SEL_8] (rows=242 width=10)
-  Output:["_col1"]
-  Filter Operator [FIL_24] (rows=242 width=10)
-predicate:value is not null
-TableScan [TS_6] (rows=242 width=10)
-  default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"]
+Select Operator [SEL_15] (rows=292 width=10)
+  Output:["_col0","_col1"]
+  Merge Join Operator [MERGEJOIN_27] (rows=292 width=10)
+Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1","_col2"]
+  <-Map 1 [SIMPLE_EDGE] llap
+SHUFFLE [RS_12]
+  PartitionCols:_col2
+  Merge Join Operator [MERGEJOIN_25] (rows=266 width=10)
+Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col1","_col2"]
+  <-Select Operator [SEL_5] (rows=242 width=10)
+  Output:["_col0","_col1"]
+  Filter Operator [FIL_23] (rows=242 width=10)
+predicate:(key is not null and value is not null)
+TableScan [TS_3] (rows=242 width=10)
+  
default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+  <-Select Operator [SEL_2] (rows=242 width=10)
+  Output:["_col0"]
+  Filter Operator [FIL_22] (rows=242 width=10)
+predicate:key is not null
+TableScan [TS_0] (rows=242 width=10)
+  default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"]
+  <-Map 4 [SIMPLE_EDGE] llap
+SHUFFLE [RS_13]
+  PartitionCols:_col1
+  Select Operator [SEL_8] (rows=242 width=10)
+Output:["_col1"]
+Filter Operator [FIL_24] (rows=242 width=10)
+  predicate:value is not null
+  TableScan [TS_6] (rows=242 width=10)
+default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"]
 
 PREHOOK: query: explain 
 select s1.key as key, s1.value as value from tab s1 join tab2 s3 on 
s1.key=s3.key
@@ -1958,34 +1960,36 @@ Stage-0
 <-Reducer 2 [CONTAINS] llap
   Reduce Output Operator [RS_24]
 PartitionCols:_col0
-Merge Join Operator [MERGEJOIN_50] (rows=292 width=10)
-  Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"]
-<-Map 1 [SIMPLE_EDGE] llap
-  SHUFFLE [RS_12]
-PartitionCols:_col1
-Merge Join Operator [MERGEJOIN_48] (rows=266 width=10)
-  
Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"]
-<-Select Operator [SEL_5] (rows=242 width=10)
-Output:["_col0"]
-Filter Operator [FIL_44] (rows=242 width=10)
-  predicate:key is not null
-  TableScan [TS_3] (rows=242 width=10)
-
default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"]
-<-Select Operator [SEL_2] (rows=242 width=10)
-Output:["_col0","_col1"]
- 

[37/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/count.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/count.q.out 
b/ql/src/test/results/clientpositive/llap/count.q.out
index 0ef26e5..81b94e7 100644
--- a/ql/src/test/results/clientpositive/llap/count.q.out
+++ b/ql/src/test/results/clientpositive/llap/count.q.out
@@ -49,22 +49,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: abcd
-  Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: a (type: int), b (type: int), c (type: int), 
d (type: int)
 outputColumnNames: a, b, c, d
-Statistics: Num rows: 4 Data size: 78 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4 Data size: 64 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count(DISTINCT b), count(DISTINCT c), 
sum(d)
   keys: a (type: int), b (type: int), c (type: int)
   mode: hash
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
-  Statistics: Num rows: 4 Data size: 78 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4 Data size: 64 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int), _col1 (type: int), 
_col2 (type: int)
 sort order: +++
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 4 Data size: 78 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4 Data size: 64 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col5 (type: bigint)
 Execution mode: llap
 LLAP IO: no inputs
@@ -76,10 +76,10 @@ STAGE PLANS:
 keys: KEY._col0 (type: int)
 mode: mergepartial
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 2 Data size: 39 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 2 Data size: 39 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE 
Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -123,21 +123,21 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: abcd
-  Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: a (type: int), b (type: int), c (type: int), 
d (type: int)
 outputColumnNames: a, b, c, d
-Statistics: Num rows: 4 Data size: 78 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4 Data size: 64 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count(), count(a), count(b), count(c), 
count(d), count(DISTINCT a), count(DISTINCT b), count(DISTINCT c), 
count(DISTINCT d), count(DISTINCT a, b), count(DISTINCT b, c), count(DISTINCT 
c, d), count(DISTINCT a, d), count(DISTINCT a, c), count(DISTINCT b, d), 
count(DISTINCT a, b, c), count(DISTINCT b, c, d), count(DISTINCT a, c, d), 
count(DISTINCT a, b, d), count(DISTINCT a, b, c, d)
   keys: a (type: int), b (type: int), c (type: int), d 
(type: int)
   mode: hash
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, 
_col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
-  Statistics: Num rows: 4 Data size: 78 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4 Data size: 64 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int), _col1 (type: int), 
_col2 (type: int), _col3 

[46/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out 
b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out
index 9e60536..60c35fa 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out
@@ -84,63 +84,66 @@ STAGE PLANS:
  A masked pattern was here 
   Edges:
 Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: b
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  alias: c
+  Statistics: Num rows: 500 Data size: 45500 Basic stats: 
COMPLETE Column stats: COMPLETE
   Filter Operator
-predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+predicate: value is not null (type: boolean)
+Statistics: Num rows: 500 Data size: 45500 Basic stats: 
COMPLETE Column stats: COMPLETE
 Select Operator
-  expressions: key (type: int)
+  expressions: value (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 45500 Basic stats: 
COMPLETE Column stats: COMPLETE
+  Reduce Output Operator
+key expressions: _col0 (type: string)
+sort order: +
+Map-reduce partition columns: _col0 (type: string)
+Statistics: Num rows: 500 Data size: 45500 Basic 
stats: COMPLETE Column stats: COMPLETE
+Execution mode: llap
+LLAP IO: no inputs
+Map 5 
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 89488 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (key is not null and value is not null) (type: 
boolean)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 450 Data size: 80539 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
-  Merge Join Operator
-condition map:
- Inner Join 0 to 1
-keys:
-  0 _col0 (type: int)
-  1 _col0 (type: int)
-outputColumnNames: _col1
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
-Reduce Output Operator
-  key expressions: _col1 (type: string)
-  sort order: +
-  Map-reduce partition columns: _col1 (type: string)
-  Statistics: Num rows: 550 Data size: 5843 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 450 Data size: 80539 Basic stats: 
COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col1 (type: string)
+sort order: +
+Map-reduce partition columns: _col1 (type: string)
+Statistics: Num rows: 450 Data size: 80539 Basic 
stats: COMPLETE Column stats: NONE
+value expressions: _col0 (type: int)
 Execution mode: llap
-Map 5 
+LLAP IO: no inputs
+Map 6 
 Map Operator Tree:
 TableScan
-  alias: c
-  Statistics: Num rows: 500 Data size: 45500 Basic stats: 
COMPLETE Column stats: COMPLETE
+  alias: b
+  Statistics: Num rows: 500 Data size: 

[63/74] [abbrv] hive git commit: HIVE-17107: Upgrade Yetus to 0.5.0 (Barna Zsombor Klara via Peter Vary)

[weiz]

HIVE-17107: Upgrade Yetus to 0.5.0 (Barna Zsombor Klara via Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a5381cb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a5381cb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a5381cb

Branch: refs/heads/hive-14535
Commit: 9a5381cb942b0a12aad3abe83f03b4cbe5ddc92f
Parents: 1923e21
Author: Peter Vary 
Authored: Wed Sep 6 13:35:29 2017 +0200
Committer: Peter Vary 
Committed: Wed Sep 6 13:35:29 2017 +0200

--
 dev-support/checkstyle_YETUS-484.sh | 406 --
 dev-support/findbugs_YETUS-471.sh   | 488 --
 dev-support/maven_YETUS-506.sh  | 687 ---
 dev-support/yetus-wrapper.sh|  11 +-
 4 files changed, 1 insertion(+), 1591 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9a5381cb/dev-support/checkstyle_YETUS-484.sh
--
diff --git a/dev-support/checkstyle_YETUS-484.sh 
b/dev-support/checkstyle_YETUS-484.sh
deleted file mode 100644
index e297a13..000
--- a/dev-support/checkstyle_YETUS-484.sh
+++ /dev/null
@@ -1,406 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_test_type checkstyle
-
-CHECKSTYLE_TIMER=0
-CHECKSTYLE_GOAL_DEFAULT="checkstyle"
-CHECKSTYLE_GOAL="${CHECKSTYLE_GOAL_DEFAULT}"
-CHECKSTYLE_OPTIONS_DEFAULT="-Dcheckstyle.consoleOutput=true"
-CHECKSTYLE_OPTIONS="${CHECKSTYLE_OPTIONS_DEFAULT}"
-
-function checkstyle_filefilter
-{
-  local filename=$1
-
-  if [[ ${BUILDTOOL} == maven
-|| ${BUILDTOOL} == ant ]]; then
-if [[ ${filename} =~ \.java$ ]]; then
-  add_test checkstyle
-fi
-  fi
-}
-
-## @description  usage help for checkstyle
-## @audience private
-## @stabilityevolving
-## @replaceable  no
-function checkstyle_usage
-{
-  yetus_add_option "--checkstyle-goal=" "Checkstyle maven plugin goal to 
use, 'check' and 'checkstyle' supported. Defaults to 
'${CHECKSTYLE_GOAL_DEFAULT}'."
-}
-
-## @description  parse checkstyle args
-## @audience private
-## @stabilityevolving
-## @replaceable  no
-## @paramarg
-## @param..
-function checkstyle_parse_args
-{
-  local i
-
-  for i in "$@"; do
-case ${i} in
---checkstyle-goal=*)
-  CHECKSTYLE_GOAL=${i#*=}
-case ${CHECKSTYLE_GOAL} in
-check)
-CHECKSTYLE_OPTIONS="-Dcheckstyle.consoleOutput=true 
-Dcheckstyle.failOnViolation=false"
-;;
-checkstyle)
-;;
-*)
-yetus_error "Warning: checkstyle goal ${CHECKSTYLE_GOAL} not 
supported. It may have unexpected behavior"
-;;
-esac
-;;
-esac
-  done
-}
-
-## @description  initialize the checkstyle plug-in
-## @audience private
-## @stabilityevolving
-## @replaceable  no
-function checkstyle_initialize
-{
-  if declare -f maven_add_install >/dev/null 2>&1; then
-maven_add_install checkstyle
-  fi
-}
-
-## @description  checkstyle plug-in specific difference calculator
-## @audience private
-## @stabilityevolving
-## @replaceable  no
-## @parambranchlog
-## @parampatchlog
-## @return   differences
-function checkstyle_calcdiffs
-{
-  declare orig=$1
-  declare new=$2
-  declare tmp=${PATCH_DIR}/pl.$$.${RANDOM}
-  declare j
-
-  # first, strip filenames:line:
-  # this keeps column: in an attempt to increase
-  # accuracy in case of multiple, repeated errors
-  # since the column number shouldn't change
-  # if the line of code hasn't been touched
-  # remove the numbers from the error message for comparing
-  # so if only the error message numbers change
-  # we do not report new error
-  # shellcheck disable=SC2016
-  cut -f3- -d: "${orig}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print 
$1":"$2}' > "${tmp}.branch"
-  # shellcheck disable=SC2016
-  cut -f3- -d: "${new}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print 
$1":"$2}' > "${tmp}.patch"
-
-  # compare the errors, generating a string of line

[60/74] [abbrv] hive git commit: HIVE-17265: Cache merged column stats from retrieved partitions (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum)

[weiz]

HIVE-17265: Cache merged column stats from retrieved partitions (Jesus Camacho 
Rodriguez, reviewed by Ashutosh Chauhan) (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/21465d11
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/21465d11
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/21465d11

Branch: refs/heads/hive-14535
Commit: 21465d11b0af976f1b16eceb79e6a097ea13
Parents: 5f13f28
Author: Jesus Camacho Rodriguez 
Authored: Tue Sep 5 17:29:38 2017 -0700
Committer: Jesus Camacho Rodriguez 
Committed: Tue Sep 5 17:29:38 2017 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/21465d11/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 3041968..17d9f2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -1096,7 +1096,7 @@ public class StatsUtils {
   stats = convertColStats(colStat, tabName);
 } catch (HiveException e) {
   LOG.error("Failed to retrieve table statistics: ", e);
-  stats = null;
+  stats = new ArrayList();
 }
 // Merge stats from cache with metastore cache
 if (colStatsCache != null) {

[41/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out 
b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
index aab4343..dd67a55 100644
--- a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
@@ -142,22 +142,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 26 Data size: 4888 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 null sort order: a
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
 tag: 0
 value expressions: _col1 (type: string)
 auto parallelism: true
@@ -218,22 +218,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 26 Data size: 4888 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 null sort order: a
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 26 Data size: 2750 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25 Data size: 4700 Basic stats: 
COMPLETE Column stats: NONE
 tag: 1
 value expressions: _col1 (type: string)
 auto parallelism: true
@@ -302,17 +302,17 @@ STAGE PLANS:
   1 _col0 (type: int)
 outputColumnNames: _col0, _col1, _col3
 Position of Big Table: 0
-Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: UDFToString(_col0) (type: string), _col1 (type: 
string), _col3 (type: string)
   outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 28 Data size: 3025 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 27 Data size: 5170 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
 GlobalTableId: 1
  A masked pattern was here 
 NumFilesPerFileSink: 1
-Statistics: Num rows: 28 Data size: 3025 Basic stats: 
COMPLETE Column stats: 

[15/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/semijoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/semijoin.q.out 
b/ql/src/test/results/clientpositive/llap/semijoin.q.out
index 5786673..f788ae3 100644
--- a/ql/src/test/results/clientpositive/llap/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/semijoin.q.out
@@ -141,15 +141,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 11 Data size: 2068 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 11 Data size: 79 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 2068 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: int)
   sort order: +
   Map-reduce partition columns: key (type: int)
-  Statistics: Num rows: 11 Data size: 79 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 11 Data size: 2068 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: value (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -157,24 +157,24 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 11 Data size: 84 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 11 Data size: 84 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
   Group By Operator
 keys: _col0 (type: int)
 mode: hash
 outputColumnNames: _col0
-Statistics: Num rows: 11 Data size: 84 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 11 Data size: 84 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -187,21 +187,21 @@ STAGE PLANS:
   0 key (type: int)
   1 _col0 (type: int)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 12 Data size: 2274 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int), _col1 (type: string)
   sort order: ++
-  Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 12 Data size: 2274 Basic stats: 
COMPLETE Column stats: NONE
 Reducer 3 
 Execution mode: llap
 Reduce Operator Tree:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: int), 
KEY.reducesinkkey1 (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 12 Data size: 2274 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 12 Data size: 2274 Basic stats: 
COMPLETE Column stats: NONE
   table:

[38/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out 
b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out
index 0f8ce65..edf8506 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out
@@ -76,57 +76,57 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: x
-  Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 5 
 Map Operator Tree:
 TableScan
   alias: y
-  Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 6 
 Map Operator Tree:
 TableScan
   alias: z
-  Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 7 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 7 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -141,18 +141,18 @@ STAGE PLANS:
   1 _col0 (type: int)
   2 _col0 (type: int)

[71/74] [abbrv] hive git commit: HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu)

[weiz]

HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, 
reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8482c5fb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8482c5fb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8482c5fb

Branch: refs/heads/hive-14535
Commit: 8482c5fbe9d2b9a62132e0e94d5578a5eaa22fbd
Parents: c52aba1
Author: Aihua Xu 
Authored: Thu Sep 7 17:17:14 2017 -0700
Committer: Aihua Xu 
Committed: Thu Sep 7 17:17:14 2017 -0700

--
 jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8482c5fb/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
--
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java 
b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index b743b46..c6bd41f 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -263,7 +263,7 @@ public class HiveStatement implements java.sql.Statement {
 TGetOperationStatusResp status = waitForOperationToComplete();
 
 // The query should be completed by now
-if (!status.isHasResultSet()) {
+if (!status.isHasResultSet() && !stmtHandle.isHasResultSet()) {
   return false;
 }
 resultSet =  new 
HiveQueryResultSet.Builder(this).setClient(client).setSessionHandle(sessHandle)

[12/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out 
b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out
index fbfc902..9079538 100644
--- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out
@@ -68,19 +68,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 2 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -88,19 +88,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 1 Data size: 206 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -108,19 +108,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: c
-  Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 376 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 2 Data size: 222 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 376 Basic 

[30/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out 
b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 788854a..6488e9e 100644
--- 
a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ 
b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -92,19 +92,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: acid_part
-  Statistics: Num rows: 1600 Data size: 30800 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1600 Data size: 150327 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (key = 'foo') (type: boolean)
-Statistics: Num rows: 800 Data size: 15400 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5 Data size: 469 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: ROW__ID (type: 
struct)
   outputColumnNames: _col0
-  Statistics: Num rows: 800 Data size: 15400 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5 Data size: 469 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: 
struct)
 sort order: +
 Map-reduce partition columns: UDFToInteger(_col0) 
(type: int)
-Statistics: Num rows: 800 Data size: 15400 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 5 Data size: 469 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: may be used (ACID table)
 Reducer 2 
@@ -113,10 +113,10 @@ STAGE PLANS:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: 
struct), 'foo' (type: string), 
'bar' (type: string), '2008-04-08' (type: string)
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 800 Data size: 15400 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 800 Data size: 15400 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE 
Column stats: NONE
   table:
   input format: 
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -185,19 +185,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: acid_part
-  Statistics: Num rows: 1600 Data size: 312400 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 1600 Data size: 444727 Basic stats: 
COMPLETE Column stats: PARTIAL
   Filter Operator
 predicate: (key = 'foo') (type: boolean)
-Statistics: Num rows: 800 Data size: 147200 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 5 Data size: 1355 Basic stats: 
COMPLETE Column stats: PARTIAL
 Select Operator
   expressions: ROW__ID (type: 
struct), ds (type: string)
   outputColumnNames: _col0, _col3
-  Statistics: Num rows: 800 Data size: 347200 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 5 Data size: 2170 Basic stats: 
COMPLETE Column stats: PARTIAL
   Reduce Output Operator
 key expressions: _col0 (type: 
struct)
 sort order: +
 Map-reduce partition columns: UDFToInteger(_col0) 
(type: int)
-Statistics: Num rows: 800 Data size: 347200 Basic 
stats: COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 5 Data size: 2170 Basic stats: 
COMPLETE Column stats: PARTIAL
 value expressions: _col3 (type: string)
 Execution mode: llap
 LLAP IO: may be used (ACID table)
@@ -207,10 +207,10 @@ 

[66/74] [abbrv] hive git commit: HIVE-17455: External LLAP client: connection to HS2 should be kept open until explicitly closed (Jason Dere, reviewed by Sergey Shelukhin)

[weiz]

HIVE-17455: External LLAP client: connection to HS2 should be kept open until 
explicitly closed (Jason Dere, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/849fa02c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/849fa02c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/849fa02c

Branch: refs/heads/hive-14535
Commit: 849fa02c97e4eba2c443db871c1e6b117750d868
Parents: aa2557b
Author: Jason Dere 
Authored: Wed Sep 6 15:00:00 2017 -0700
Committer: Jason Dere 
Committed: Wed Sep 6 15:00:00 2017 -0700

--
 .../apache/hive/jdbc/TestJdbcWithMiniLlap.java  |   5 +
 .../hadoop/hive/llap/LlapBaseInputFormat.java   | 106 +--
 2 files changed, 100 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/849fa02c/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
index b5f1f13..68d2ddc 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
@@ -44,6 +44,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
@@ -143,6 +144,7 @@ public class TestJdbcWithMiniLlap {
 
   @After
   public void tearDown() throws Exception {
+LlapBaseInputFormat.closeAll();
 hs2Conn.close();
   }
 
@@ -475,6 +477,7 @@ public class TestJdbcWithMiniLlap {
 String url = miniHS2.getJdbcURL();
 String user = System.getProperty("user.name");
 String pwd = user;
+String handleId = UUID.randomUUID().toString();
 
 LlapRowInputFormat inputFormat = new LlapRowInputFormat();
 
@@ -484,6 +487,7 @@ public class TestJdbcWithMiniLlap {
 job.set(LlapBaseInputFormat.USER_KEY, user);
 job.set(LlapBaseInputFormat.PWD_KEY, pwd);
 job.set(LlapBaseInputFormat.QUERY_KEY, query);
+job.set(LlapBaseInputFormat.HANDLE_ID, handleId);
 
 InputSplit[] splits = inputFormat.getSplits(job, numSplits);
 assertTrue(splits.length > 0);
@@ -503,6 +507,7 @@ public class TestJdbcWithMiniLlap {
   }
   reader.close();
 }
+LlapBaseInputFormat.close(handleId);
 
 return rowCount;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/849fa02c/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
--
diff --git 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java 
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
index f2d9074..215f5b1 100644
--- 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
+++ 
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
@@ -31,8 +31,11 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.apache.commons.collections4.ListUtils;
@@ -70,6 +73,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hive.common.util.ShutdownHookManager;
 import org.apache.tez.common.security.JobTokenIdentifier;
 import org.apache.tez.common.security.TokenCache;
 import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -92,6 +96,8 @@ public class LlapBaseInputFormat>
   private static final Logger LOG = 
LoggerFactory.getLogger(LlapBaseInputFormat.class);
 
   private static String driverName = "org.apache.hive.jdbc.HiveDriver";
+  private static final Map connectionMap = new 
ConcurrentHashMap();
+
   private String url;  // "jdbc:hive2://localhost:1/default"
   private String user; // "hive",
   private String pwd;  // ""
@@ -102,6 +108,7 @@ public class LlapBaseInputFormat>
   public static final String QUERY_KEY = "llap.if.query";
   public static final String USER_KEY = "llap.if.user";
   public static final String PWD_KEY = "llap.if.pwd";
+  public static final 

[09/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out 
b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 97e1216..92786f7 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -97,19 +97,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_null
-  Statistics: Num rows: 5 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (p_size is not null and p_brand is not null) 
(type: boolean)
-Statistics: Num rows: 5 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: p_partkey (type: int), p_name (type: 
string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), 
p_size (type: int), p_container (type: string), p_retailprice (type: double), 
p_comment (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8
-  Statistics: Num rows: 5 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col5 (type: int)
 sort order: +
 Map-reduce partition columns: _col5 (type: int)
-Statistics: Num rows: 5 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: int), _col1 (type: 
string), _col2 (type: string), _col3 (type: string), _col4 (type: string), 
_col6 (type: string), _col7 (type: double), _col8 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -117,44 +117,44 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_null
-  Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 814 Data size: 3100 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: p_size is not null (type: boolean)
-Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 774 Data size: 2947 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   keys: p_size (type: int)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 774 Data size: 2947 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 774 Data size: 2947 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 6 
 Map Operator Tree:
 TableScan
   alias: part_null
-  Statistics: Num rows: 32 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 32 Data size: 5888 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: p_brand is not null (type: boolean)
-Statistics: Num rows: 32 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 31 Data size: 5704 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: p_brand (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 32 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 31 Data size: 5704 Basic stats: 
COMPLETE Column stats: NONE
 

[21/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out 
b/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out
index e5497d5..6d20939 100644
--- a/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out
+++ b/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out
@@ -61,23 +61,23 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: src_10
-  Statistics: Num rows: 10 Data size: 104 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Lateral View Forward
-Statistics: Num rows: 10 Data size: 104 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: key
-  Statistics: Num rows: 10 Data size: 104 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Lateral View Join Operator
 outputColumnNames: _col0, _col5
-Statistics: Num rows: 20 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 20 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: _col0 (type: string), 
UDFToString(_col5) (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 20 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 20 Data size: 3680 Basic 
stats: COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 20 Data size: 208 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 20 Data size: 3680 Basic 
stats: COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.TextInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -86,41 +86,41 @@ STAGE PLANS:
 Select Operator
   expressions: array((key + 1),(key + 2)) (type: 
array)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 104 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   UDTF Operator
-Statistics: Num rows: 10 Data size: 104 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
 function name: explode
 Lateral View Join Operator
   outputColumnNames: _col0, _col5
-  Statistics: Num rows: 20 Data size: 208 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 20 Data size: 3680 Basic 
stats: COMPLETE Column stats: NONE
   Select Operator
 expressions: _col0 (type: string), 
UDFToString(_col5) (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 20 Data size: 208 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 20 Data size: 3680 Basic 
stats: COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 20 Data size: 208 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 20 Data size: 3680 Basic 
stats: COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.TextInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
   serde: 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
   name: default.src_lv1
   Lateral View Forward
-  

[25/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out 
b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
index 430bdaf..2a89123 100644
--- a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out
@@ -43,22 +43,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: src_orc
-  Statistics: Num rows: 10 Data size: 3560 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 7360 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: false (type: boolean)
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string), value (type: string), 
ds (type: string), hr (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3
-  Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
 GlobalTableId: 0
  A masked pattern was here 
 NumFilesPerFileSink: 1
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
  A masked pattern was here 
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -165,23 +165,23 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: src_orc
-  Statistics: Num rows: 10 Data size: 3560 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: false (type: boolean)
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 10 Data size: 3560 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
 Limit
   Number of rows: 0
   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE 
Column stats: NONE
@@ -201,10 +201,10 @@ STAGE PLANS:
   0 _col0 (type: string)
   1 _col0 (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 1 Data size: 391 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 1 Data size: 391 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE 
Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -262,18 +262,18 @@ 

[49/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out 
b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
index f04519f..2ac67b2 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
@@ -69,22 +69,6 @@ STAGE PLANS:
 Map Reduce
   Map Operator Tree:
   TableScan
-alias: p1
-Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
-Filter Operator
-  predicate: p_name is not null (type: boolean)
-  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
-  Select Operator
-expressions: p_partkey (type: int), p_name (type: string), 
p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size 
(type: int), p_container (type: string), p_retailprice (type: double), 
p_comment (type: string)
-outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8
-Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
-Reduce Output Operator
-  key expressions: _col1 (type: string)
-  sort order: +
-  Map-reduce partition columns: _col1 (type: string)
-  Statistics: Num rows: 26 Data size: 3147 Basic stats: 
COMPLETE Column stats: NONE
-  value expressions: _col0 (type: int), _col2 (type: string), 
_col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: 
string), _col7 (type: double), _col8 (type: string)
-  TableScan
 alias: p2
 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
 Filter Operator
@@ -117,6 +101,22 @@ STAGE PLANS:
   Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
   value expressions: _col0 (type: int), _col2 (type: string), 
_col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: 
string), _col7 (type: double), _col8 (type: string)
   TableScan
+alias: p1
+Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
+Filter Operator
+  predicate: p_name is not null (type: boolean)
+  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
+  Select Operator
+expressions: p_partkey (type: int), p_name (type: string), 
p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size 
(type: int), p_container (type: string), p_retailprice (type: double), 
p_comment (type: string)
+outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8
+Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
+Reduce Output Operator
+  key expressions: _col1 (type: string)
+  sort order: +
+  Map-reduce partition columns: _col1 (type: string)
+  Statistics: Num rows: 26 Data size: 3147 Basic stats: 
COMPLETE Column stats: NONE
+  value expressions: _col0 (type: int), _col2 (type: string), 
_col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: 
string), _col7 (type: double), _col8 (type: string)
+  TableScan
 alias: p4
 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE 
Column stats: NONE
 Filter Operator
@@ -136,8 +136,8 @@ STAGE PLANS:
 Join Operator
   condition map:
Inner Join 0 to 1
-   Inner Join 1 to 2
-   Inner Join 0 to 3
+   Inner Join 0 to 2
+   Inner Join 2 to 3
   keys:
 0 _col1 (type: string)
 1 _col1 (type: string)
@@ -145,13 +145,17 @@ STAGE PLANS:
 3 _col1 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, 
_col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, 
_col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35
   Statistics: Num rows: 85 Data size: 10385 Basic stats: COMPLETE 
Column stats: NONE
-  File Output Operator
-compressed: false
+  Select Operator
+expressions: _col18 (type: int), _col19 (type: string), _col20 
(type: string), _col21 (type: string), _col22 (type: string), _col23 (type: 
int), _col24 (type: 

[22/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out 
b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out
index 6d756a8..58c29fe 100644
--- a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out
+++ b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out
@@ -192,14 +192,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: bigtbl
-  Statistics: Num rows: 5000 Data size: 53120 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5000 Data size: 1748368 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (key is not null and value is not null) (type: 
boolean)
-Statistics: Num rows: 5000 Data size: 53120 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4500 Data size: 1573531 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 5000 Data size: 53120 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4500 Data size: 1573531 Basic 
stats: COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -209,11 +209,11 @@ STAGE PLANS:
 outputColumnNames: _col1
 input vertices:
   1 Map 3
-Statistics: Num rows: 5500 Data size: 58432 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 4950 Data size: 1730884 Basic 
stats: COMPLETE Column stats: NONE
 Select Operator
   expressions: _col1 (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 5500 Data size: 58432 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 4950 Data size: 1730884 Basic 
stats: COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -222,7 +222,7 @@ STAGE PLANS:
   1 _col0 (type: string)
 input vertices:
   1 Map 4
-Statistics: Num rows: 6050 Data size: 64275 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 5445 Data size: 1903972 
Basic stats: COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   mode: hash
@@ -238,38 +238,38 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: smalltbl1
-  Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Map 4 
 Map Operator Tree:
 TableScan
   alias: smalltbl2
-  Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1840 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 

[65/74] [abbrv] hive git commit: HIVE-17393 : AMReporter need hearbeat every external 'AM' (Zhiyuan Yang, reviewed by Sergey Shelukhin)

[weiz]

HIVE-17393 : AMReporter need hearbeat every external 'AM' (Zhiyuan Yang, 
reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aa2557b2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aa2557b2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aa2557b2

Branch: refs/heads/hive-14535
Commit: aa2557b24f4c77638315144cbb6d1c72c04c0f24
Parents: 660e39e
Author: sergey 
Authored: Wed Sep 6 12:30:55 2017 -0700
Committer: sergey 
Committed: Wed Sep 6 12:30:55 2017 -0700

--
 .../hive/llap/daemon/impl/AMReporter.java   | 75 ++---
 .../daemon/impl/comparator/TestAMReporter.java  | 87 
 2 files changed, 134 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/aa2557b2/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
--
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index b4c62d5..b784360 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -111,7 +111,7 @@ public class AMReporter extends AbstractService {
   private final AtomicBoolean isShutdown = new AtomicBoolean(false);
   // Tracks appMasters to which heartbeats are being sent. This should not be 
used for any other
   // messages like taskKilled, etc.
-  private final Map knownAppMasters = new 
HashMap<>();
+  private final Map> 
knownAppMasters = new HashMap<>();
   volatile ListenableFuture queueLookupFuture;
   private final DaemonId daemonId;
 
@@ -208,11 +208,16 @@ public class AMReporter extends AbstractService {
 // and discard AMNodeInfo instances per query.
 synchronized (knownAppMasters) {
   LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
-  amNodeInfo = knownAppMasters.get(queryIdentifier);
+  Map amNodeInfoPerQuery = 
knownAppMasters.get(queryIdentifier);
+  if (amNodeInfoPerQuery == null) {
+amNodeInfoPerQuery = new HashMap<>();
+knownAppMasters.put(queryIdentifier, amNodeInfoPerQuery);
+  }
+  amNodeInfo = amNodeInfoPerQuery.get(amNodeId);
   if (amNodeInfo == null) {
-amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, 
queryIdentifier,
-retryPolicy, retryTimeout, socketFactory, conf);
-knownAppMasters.put(queryIdentifier, amNodeInfo);
+amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, 
queryIdentifier, retryPolicy,
+  retryTimeout, socketFactory, conf);
+amNodeInfoPerQuery.put(amNodeId, amNodeInfo);
 // Add to the queue only the first time this is registered, and on
 // subsequent instances when it's taken off the queue.
 amNodeInfo.setNextHeartbeatTime(System.currentTimeMillis() + 
heartbeatInterval);
@@ -233,7 +238,7 @@ public class AMReporter extends AbstractService {
 }
 AMNodeInfo amNodeInfo;
 synchronized (knownAppMasters) {
-  amNodeInfo = knownAppMasters.get(queryIdentifier);
+  amNodeInfo = getAMNodeInfo(amLocation, port, queryIdentifier);
   if (amNodeInfo == null) {
 LOG.info(("Ignoring duplicate unregisterRequest for am at: " + 
amLocation + ":" + port));
   } else {
@@ -249,7 +254,7 @@ public class AMReporter extends AbstractService {
 LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
 AMNodeInfo amNodeInfo;
 synchronized (knownAppMasters) {
-  amNodeInfo = knownAppMasters.get(queryIdentifier);
+  amNodeInfo = getAMNodeInfo(amLocation, port, queryIdentifier);
   if (amNodeInfo == null) {
 amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, 
queryIdentifier, retryPolicy, retryTimeout, socketFactory,
   conf);
@@ -277,20 +282,22 @@ public class AMReporter extends AbstractService {
   public void queryComplete(QueryIdentifier queryIdentifier) {
 if (queryIdentifier != null) {
   synchronized (knownAppMasters) {
-AMNodeInfo amNodeInfo = knownAppMasters.remove(queryIdentifier);
+LOG.debug("Query complete received for {}", queryIdentifier);
+Map amNodeInfoPerQuery = 
knownAppMasters.remove(queryIdentifier);
 
 // The AM can be used for multiple queries. This is an indication that 
a single query is complete.
 // We don't have a good mechanism to know when an app ends. Removing 
this right now ensures
 // that a new one 

[54/74] [abbrv] hive git commit: HIVE-17225: HoS DPP pruning sink ops can target parallel work objects (Sahil Takiar, reviewed by Sergio Pena)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/90e68288/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
index f2403aa..7df1b9e 100644
--- 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
+++ 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
@@ -3584,6 +3584,25 @@ STAGE PLANS:
 Map 1 
 Map Operator Tree:
 TableScan
+  alias: srcpart_hour
+  filterExpr: (UDFToDouble(hr) = 13.0) (type: boolean)
+  Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE 
Column stats: NONE
+  Filter Operator
+predicate: (UDFToDouble(hr) = 13.0) (type: boolean)
+Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
+Select Operator
+  expressions: hr (type: string)
+  outputColumnNames: _col0
+  Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col0 (type: string)
+sort order: +
+Map-reduce partition columns: _col0 (type: string)
+Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
+Execution mode: vectorized
+Map 5 
+Map Operator Tree:
+TableScan
   alias: srcpart
   filterExpr: ((UDFToDouble(hr) = 13.0) and ds is not null) 
(type: boolean)
   Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
@@ -3595,13 +3614,13 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1
   Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
   Reduce Output Operator
-key expressions: _col0 (type: string)
+key expressions: _col1 (type: string)
 sort order: +
-Map-reduce partition columns: _col0 (type: string)
+Map-reduce partition columns: _col1 (type: string)
 Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
-value expressions: _col1 (type: string)
+value expressions: _col0 (type: string)
 Execution mode: vectorized
-Map 5 
+Map 6 
 Map Operator Tree:
 TableScan
   alias: srcpart_date
@@ -3620,25 +3639,6 @@ STAGE PLANS:
 Map-reduce partition columns: _col0 (type: string)
 Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: vectorized
-Map 6 
-Map Operator Tree:
-TableScan
-  alias: srcpart_hour
-  filterExpr: (UDFToDouble(hr) = 13.0) (type: boolean)
-  Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE 
Column stats: NONE
-  Filter Operator
-predicate: (UDFToDouble(hr) = 13.0) (type: boolean)
-Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
-Select Operator
-  expressions: hr (type: string)
-  outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
-  Reduce Output Operator
-key expressions: _col0 (type: string)
-sort order: +
-Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 1 Data size: 172 Basic stats: 
COMPLETE Column stats: NONE
-Execution mode: vectorized
 Reducer 2 
 Reduce Operator Tree:
   Join Operator
@@ -3646,14 +3646,14 @@ STAGE PLANS:
  Inner Join 0 to 1
 keys:
   0 _col0 (type: string)
-  1 _col0 (type: string)
+  1 _col1 (type: string)
 outputColumnNames: _col1
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
  

[39/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/column_access_stats.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/column_access_stats.q.out 
b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out
index c56c818..4cdb4861 100644
--- a/ql/src/test/results/clientpositive/llap/column_access_stats.q.out
+++ b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out
@@ -185,14 +185,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t1
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 2 Data size: 60 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -203,14 +203,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t1
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: val (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 2 Data size: 60 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -273,14 +273,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t1
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 2 Data size: 60 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -291,14 +291,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t1
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 2 Data size: 60 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   

[18/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out
 
b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out
index 22c7745..3613da5 100644
--- 
a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out
+++ 
b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out
@@ -155,14 +155,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_change_various_various_struct1
-  Statistics: Num rows: 6 Data size: 4734 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 6 Data size: 5862 Basic stats: 
COMPLETE Column stats: PARTIAL
   Select Operator
 expressions: insert_num (type: int), part (type: int), s1 
(type: 
struct),
 b (type: string)
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 6 Data size: 24 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: PARTIAL
 File Output Operator
   compressed: false
-  Statistics: Num rows: 6 Data size: 24 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: PARTIAL
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -431,14 +431,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_add_various_various_struct2
-  Statistics: Num rows: 8 Data size: 4912 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 8 Data size: 6416 Basic stats: 
COMPLETE Column stats: PARTIAL
   Select Operator
 expressions: insert_num (type: int), part (type: int), b 
(type: string), s2 (type: 
struct)
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 8 Data size: 32 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 8 Data size: 1536 Basic stats: 
COMPLETE Column stats: PARTIAL
 File Output Operator
   compressed: false
-  Statistics: Num rows: 8 Data size: 32 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 8 Data size: 1536 Basic stats: 
COMPLETE Column stats: PARTIAL
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -635,14 +635,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_add_to_various_various_struct4
-  Statistics: Num rows: 4 Data size: 1172 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 4 Data size: 1924 Basic stats: 
COMPLETE Column stats: PARTIAL
   Select Operator
 expressions: insert_num (type: int), part (type: int), b 
(type: string), s3 (type: 
struct)
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 4 Data size: 16 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 4 Data size: 768 Basic stats: 
COMPLETE Column stats: PARTIAL
 File Output Operator
   compressed: false
-  Statistics: Num rows: 4 Data size: 16 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 4 Data size: 768 Basic stats: 
COMPLETE Column stats: PARTIAL
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

[74/74] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

HIVE-14671 : merge master into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/18d13385
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/18d13385
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/18d13385

Branch: refs/heads/hive-14535
Commit: 18d133857b7930242538b49f33b3ca517b2e287d
Parents: 42a3857 8482c5f
Author: Wei Zheng 
Authored: Thu Sep 7 23:34:23 2017 -0700
Committer: Wei Zheng 
Committed: Thu Sep 7 23:34:23 2017 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   23 +-
 .../apache/hadoop/hive/conf/TestHiveConf.java   |4 +
 .../hive/conf/TestHiveConfRestrictList.java |2 +-
 dev-support/checkstyle_YETUS-484.sh |  406 --
 dev-support/findbugs_YETUS-471.sh   |  488 --
 dev-support/maven_YETUS-506.sh  |  687 ---
 dev-support/yetus-wrapper.sh|   11 +-
 errata.txt  |1 +
 .../mapjoin/MapJoinMultiKeyBench.java   |   94 +-
 .../mapjoin/MapJoinOneLongKeyBench.java |   94 +-
 .../mapjoin/MapJoinOneStringKeyBench.java   |   94 +-
 .../hive/ql/parse/TestReplicationScenarios.java |  145 +
 .../apache/hive/jdbc/TestJdbcWithMiniLlap.java  |   27 +-
 .../test/resources/testconfiguration.properties |2 +
 .../apache/hive/beeline/QFileBeeLineClient.java |6 +-
 .../org/apache/hive/jdbc/HiveStatement.java |2 +-
 .../hive/llap/registry/LlapServiceInstance.java |   59 +
 .../llap/registry/LlapServiceInstanceSet.java   |   34 +
 .../hive/llap/registry/ServiceInstance.java |   86 -
 .../hive/llap/registry/ServiceInstanceSet.java  |   69 -
 .../ServiceInstanceStateChangeListener.java |   42 -
 .../hive/llap/registry/ServiceRegistry.java |8 +-
 .../registry/impl/InactiveServiceInstance.java  |4 +-
 .../registry/impl/LlapFixedRegistryImpl.java|   41 +-
 .../llap/registry/impl/LlapRegistryService.java |   18 +-
 .../impl/LlapZookeeperRegistryImpl.java |  800 +---
 .../hive/llap/security/LlapTokenClient.java |   18 +-
 .../hive/llap/tez/LlapProtocolClientProxy.java  |   25 +-
 .../hadoop/hive/registry/ServiceInstance.java   |   47 +
 .../hive/registry/ServiceInstanceSet.java   |   61 +
 .../ServiceInstanceStateChangeListener.java |   42 +
 .../hive/registry/impl/ServiceInstanceBase.java |   93 +
 .../hive/registry/impl/ZkRegistryBase.java  |  549 +++
 .../hive/registry/impl/ZookeeperUtils.java  |  116 +
 llap-common/pom.xml |5 +
 .../plugin/rpc/LlapPluginProtocolProtos.java| 1273 ++
 .../org/apache/hadoop/hive/llap/LlapUtil.java   |   70 +-
 .../llap/impl/LlapPluginProtocolClientImpl.java |   46 +
 .../hadoop/hive/llap/impl/ProtobufProxy.java|   76 +
 .../llap/protocol/LlapPluginProtocolPB.java |   27 +
 .../src/protobuf/LlapPluginProtocol.proto   |   36 +
 .../hadoop/hive/llap/LlapBaseInputFormat.java   |  141 +-
 .../hive/llap/cli/LlapStatusServiceDriver.java  |6 +-
 .../hive/llap/daemon/impl/AMReporter.java   |   75 +-
 .../daemon/impl/LlapProtocolServerImpl.java |   67 +-
 .../daemon/services/impl/LlapWebServices.java   |4 +-
 .../daemon/impl/comparator/TestAMReporter.java  |   87 +
 .../llap/tezplugins/LlapTaskCommunicator.java   |   69 +-
 .../tezplugins/LlapTaskSchedulerService.java|  734 ++-
 .../endpoint/LlapPluginPolicyProvider.java  |   34 +
 .../endpoint/LlapPluginSecurityInfo.java|   62 +
 .../endpoint/LlapPluginServerImpl.java  |   80 +
 .../org.apache.hadoop.security.SecurityInfo |   14 +
 .../TestLlapTaskSchedulerService.java   |  567 ++-
 .../TestDataSourceProviderFactory.java  |6 +
 pom.xml |8 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   15 +
 .../hadoop/hive/ql/exec/OperatorUtils.java  |   31 +
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   36 +-
 .../hive/ql/exec/spark/SparkUtilities.java  |4 +-
 .../apache/hadoop/hive/ql/exec/tez/Utils.java   |6 +-
 .../hadoop/hive/ql/log/LogDivertAppender.java   |   25 +-
 .../hive/ql/log/LogDivertAppenderForTest.java   |   18 +-
 .../ql/optimizer/calcite/RelOptHiveTable.java   |   93 +-
 .../physical/LlapClusterStateForCompile.java|8 +-
 .../SparkDynamicPartitionPruningResolver.java   |  146 +
 .../physical/SparkMapJoinResolver.java  |   35 +-
 .../spark/SparkPartitionPruningSinkDesc.java|   13 +-
 .../spark/SparkSkewJoinProcFactory.java |6 +-
 .../stats/annotation/StatsRulesProcFactory.java |   12 +-
 .../hive/ql/parse/BaseSemanticAnalyzer.java |   26 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |7 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |   14 +
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |4 +
 

[26/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/join_is_not_distinct_from.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/join_is_not_distinct_from.q.out 
b/ql/src/test/results/clientpositive/llap/join_is_not_distinct_from.q.out
index eaf09ca..ce903ea 100644
--- a/ql/src/test/results/clientpositive/llap/join_is_not_distinct_from.q.out
+++ b/ql/src/test/results/clientpositive/llap/join_is_not_distinct_from.q.out
@@ -34,12 +34,12 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Output Operator
 key expressions: key (type: int)
 sort order: +
 Map-reduce partition columns: key (type: int)
-Statistics: Num rows: 3 Data size: 26 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 3 Data size: 24 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: value (type: int)
 Execution mode: llap
 LLAP IO: no inputs
@@ -47,12 +47,12 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Output Operator
 key expressions: value (type: int)
 sort order: +
 Map-reduce partition columns: value (type: int)
-Statistics: Num rows: 3 Data size: 26 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 3 Data size: 24 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: key (type: int)
 Execution mode: llap
 LLAP IO: no inputs
@@ -67,14 +67,14 @@ STAGE PLANS:
   1 value (type: int)
 nullSafes: [true]
 outputColumnNames: _col0, _col1, _col5, _col6
-Statistics: Num rows: 3 Data size: 28 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: int), _col1 (type: int), _col5 
(type: int), _col6 (type: int)
   outputColumnNames: _col0, _col1, _col2, _col3
-  Statistics: Num rows: 3 Data size: 28 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 3 Data size: 28 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 3 Data size: 26 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -125,15 +125,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 3 Data size: 26 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 3 Data size: 24 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: int)
   sort order: +
   Map-reduce partition columns: key (type: int)
-  Statistics: Num rows: 3 Data size: 26 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 3 Data size: 24 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: value (type: int)
 Execution mode: llap
 LLAP IO: no inputs
@@ -141,15 +141,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
   

[24/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/mapjoin46.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out 
b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out
index efada10..c4ce854 100644
--- a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out
+++ b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out
@@ -62,11 +62,11 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: test1
-  Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: int), value (type: int), col_1 
(type: string)
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 6 Data size: 56 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
 Map Join Operator
   condition map:
Left Outer Join 0 to 1
@@ -76,10 +76,10 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
   input vertices:
 1 Map 2
-  Statistics: Num rows: 6 Data size: 61 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1267 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 6 Data size: 61 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1267 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -90,16 +90,16 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: test2
-  Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: key (type: int), value (type: int), col_2 
(type: string)
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 4 Data size: 38 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 4 Data size: 768 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col1 (type: int)
   sort order: +
   Map-reduce partition columns: _col1 (type: int)
-  Statistics: Num rows: 4 Data size: 38 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4 Data size: 768 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: int), _col2 (type: 
string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -162,11 +162,11 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: test1
-  Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: int), value (type: int), col_1 
(type: string)
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 6 Data size: 56 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
 Map Join Operator
   condition map:
Left Outer Join 0 to 1
@@ -179,10 +179,10 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
   input vertices:
 1 Map 2
-  Statistics: Num rows: 6 Data size: 61 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1267 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 6 Data size: 61 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 

[72/74] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
--
diff --cc ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index c9ccf23,d26a9a3..9f764b8
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@@ -509,13 -509,13 +509,13 @@@ Stage-
Conditional Operator
  Stage-1
Map 1 vectorized
 -  File Output Operator [FS_10]
 +  File Output Operator [FS_8]
  table:{"name:":"default.orc_merge5"}
- Select Operator [SEL_7] (rows=306 width=268)
+ Select Operator [SEL_9] (rows=306 width=335)
Output:["_col0","_col1","_col2","_col3","_col4"]
-   Filter Operator [FIL_6] (rows=306 width=268)
+   Filter Operator [FIL_8] (rows=306 width=335)
  predicate:(userid <= 13)
- TableScan [TS_0] (rows=919 width=268)
+ TableScan [TS_0] (rows=919 width=335)

default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"]
  Stage-4(CONDITIONAL)
File Merge

[36/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out 
b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out
index c865788..3b3d371 100644
--- a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out
@@ -49,11 +49,11 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 175168 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: string), value (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 500 Data size: 175168 Basic stats: 
COMPLETE Column stats: NONE
 Map Join Operator
   condition map:
Inner Join 0 to 1
@@ -63,10 +63,10 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1, _col2, _col3
   input vertices:
 1 Map 2
-  Statistics: Num rows: 5000 Data size: 106120 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 5000 Data size: 3596680 Basic 
stats: COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 5000 Data size: 106120 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 5000 Data size: 3596680 Basic 
stats: COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -77,14 +77,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: string), value (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 10 Data size: 96 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 10 Data size: 96 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: string), _col1 (type: 
string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -117,14 +117,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: d1
-  Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 96 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: string), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 10 Data size: 96 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 3680 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -134,10 +134,10 @@ STAGE PLANS:
 outputColumnNames: _col0, _col1, _col2, _col3
 input vertices:
   1 Map 2
-Statistics: Num rows: 11 Data size: 105 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 4048 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output 

[59/74] [abbrv] hive git commit: HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen Gangam)

[weiz]

HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen 
Gangam)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5f13f286
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5f13f286
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5f13f286

Branch: refs/heads/hive-14535
Commit: 5f13f286cfddc4ed0a06eb40b7c69a5ca69ab300
Parents: 9a09b78
Author: Aihua Xu 
Authored: Tue Aug 22 16:05:58 2017 -0700
Committer: Aihua Xu 
Committed: Tue Sep 5 11:23:49 2017 -0700

--
 .../mapjoin/MapJoinMultiKeyBench.java   | 94 +---
 .../mapjoin/MapJoinOneLongKeyBench.java | 94 +---
 .../mapjoin/MapJoinOneStringKeyBench.java   | 94 +---
 pom.xml |  6 +-
 .../hadoop/hive/ql/log/LogDivertAppender.java   | 25 +++---
 .../hive/ql/log/LogDivertAppenderForTest.java   | 18 ++--
 6 files changed, 27 insertions(+), 304 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5f13f286/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java
--
diff --git 
a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java
 
b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java
index f183bb5..7a5b721 100644
--- 
a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java
+++ 
b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java
@@ -18,107 +18,15 @@
 
 package org.apache.hive.benchmark.vectorization.mapjoin;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorBatchDebug;
-import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping;
-import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow;
-import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator;
-import 
org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
-import 
org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator;
-import 
org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType;
-import 
org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType.GenerateCategory;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol;
-import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig;
 import 
org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.MapJoinTestImplementation;
-import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestData;
-import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription;
-import 
org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastMultiKeyHashMap;
-import 
org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastTableContainer;
-import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VerifyFastRow;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
-import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;

[56/74] [abbrv] hive git commit: HIVE-17414: HoS DPP + Vectorization generates invalid explain plan due to CombineEquivalentWorkResolver (Liyun Zhang, reviewed by Sahil Takiar and Li Rui)

[weiz]

HIVE-17414: HoS DPP + Vectorization generates invalid explain plan due to 
CombineEquivalentWorkResolver (Liyun Zhang, reviewed by Sahil Takiar and Li Rui)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bff8d95e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bff8d95e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bff8d95e

Branch: refs/heads/hive-14535
Commit: bff8d95e91f94770fb0ae4a96009e22d16d7ddfa
Parents: 90e6828
Author: Ferdinand Xu 
Authored: Tue Sep 5 13:36:08 2017 +0800
Committer: Ferdinand Xu 
Committed: Tue Sep 5 13:36:08 2017 +0800

--
 .../hive/ql/exec/spark/SparkUtilities.java  |  4 +--
 ...k_vectorized_dynamic_partition_pruning.q.out | 28 
 2 files changed, 2 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bff8d95e/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index fac3cea..ca19fd0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -191,7 +191,7 @@ public class SparkUtilities {
   }
 
   /**
-   * Recursively find all operators under root, that are of class clazz, and
+   * Recursively find all operators under root, that are of class clazz or are 
the sub-class of clazz, and
* put them in result.
* @param result all operators under root that are of class clazz
* @param root the root operator under which all operators will be examined
@@ -202,7 +202,7 @@ public class SparkUtilities {
 if (root == null) {
   return;
 }
-if (clazz.equals(root.getClass())) {
+if (clazz.isAssignableFrom(root.getClass())) {
   result.add(root);
 }
 for (Operator child : root.getChildOperators()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/bff8d95e/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
index 7df1b9e..7cafb52 100644
--- 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
+++ 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
@@ -4353,20 +4353,6 @@ STAGE PLANS:
   partition key expr: ds
   Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   target work: Map 1
-Select Operator
-  expressions: _col0 (type: string)
-  outputColumnNames: _col0
-  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
-  Group By Operator
-keys: _col0 (type: string)
-mode: hash
-outputColumnNames: _col0
-Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
-Spark Partition Pruning Sink Operator
-  Target column: ds (string)
-  partition key expr: ds
-  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
-  target work: Map 4
 Reducer 13 
 Execution mode: vectorized
 Reduce Operator Tree:
@@ -4397,20 +4383,6 @@ STAGE PLANS:
   partition key expr: ds
   Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   target work: Map 1
-Select Operator
-  expressions: _col0 (type: string)
-  outputColumnNames: _col0
-  Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
-  Group By Operator
-keys: _col0 (type: string)
-mode: hash
-outputColumnNames: _col0
-Statistics: Num rows: 2 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
-Spark Partition Pruning Sink Operator
-  Target column: ds (string)
-  

[70/74] [abbrv] hive git commit: HIVE-17421: Clear incorrect stats after replication (Daniel Dai, reviewed by Anishek Agarwal, Thejas Nair)

[weiz]

HIVE-17421: Clear incorrect stats after replication (Daniel Dai, reviewed by 
Anishek Agarwal, Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c52aba1a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c52aba1a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c52aba1a

Branch: refs/heads/hive-14535
Commit: c52aba1a6bc7c983a8c5776723ce138a76b52064
Parents: 1e3e74e
Author: Daniel Dai 
Authored: Thu Sep 7 12:17:57 2017 -0700
Committer: Daniel Dai 
Committed: Thu Sep 7 12:17:57 2017 -0700

--
 .../hive/ql/parse/TestReplicationScenarios.java | 70 
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |  5 ++
 2 files changed, 75 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c52aba1a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 9667449..6a2e400 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -141,6 +141,7 @@ public class TestReplicationScenarios {
 hconf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
 hconf.set(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
   
"org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore");
+hconf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
 System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
 System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
 
@@ -2852,6 +2853,75 @@ public class TestReplicationScenarios {
 }
   }
 
+  @Test
+  public void testRemoveStats() throws IOException {
+String name = testName.getMethodName();
+String dbName = createDB(name, driver);
+
+String[] unptn_data = new String[]{ "1" , "2" };
+String[] ptn_data_1 = new String[]{ "5", "7", "8"};
+String[] ptn_data_2 = new String[]{ "3", "2", "9"};
+
+String unptn_locn = new Path(TEST_PATH, name + "_unptn").toUri().getPath();
+String ptn_locn_1 = new Path(TEST_PATH, name + "_ptn1").toUri().getPath();
+String ptn_locn_2 = new Path(TEST_PATH, name + "_ptn2").toUri().getPath();
+
+createTestDataFile(unptn_locn, unptn_data);
+createTestDataFile(ptn_locn_1, ptn_data_1);
+createTestDataFile(ptn_locn_2, ptn_data_2);
+
+run("CREATE TABLE " + dbName + ".unptned(a int) STORED AS TEXTFILE", 
driver);
+run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + 
dbName + ".unptned", driver);
+run("CREATE TABLE " + dbName + ".ptned(a int) partitioned by (b int) 
STORED AS TEXTFILE", driver);
+run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + 
dbName + ".ptned PARTITION(b=1)", driver);
+run("ANALYZE TABLE " + dbName + ".unptned COMPUTE STATISTICS FOR COLUMNS", 
driver);
+run("ANALYZE TABLE " + dbName + ".unptned COMPUTE STATISTICS", driver);
+run("ANALYZE TABLE " + dbName + ".ptned partition(b) COMPUTE STATISTICS 
FOR COLUMNS", driver);
+run("ANALYZE TABLE " + dbName + ".ptned partition(b) COMPUTE STATISTICS", 
driver);
+
+verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, 
driver);
+verifySetup("SELECT count(*) from " + dbName + ".unptned", new 
String[]{"2"}, driver);
+verifySetup("SELECT count(*) from " + dbName + ".ptned", new 
String[]{"3"}, driver);
+verifySetup("SELECT max(a) from " + dbName + ".unptned", new 
String[]{"2"}, driver);
+verifySetup("SELECT max(a) from " + dbName + ".ptned where b=1", new 
String[]{"8"}, driver);
+
+advanceDumpDir();
+run("REPL DUMP " + dbName, driver);
+String replDumpLocn = getResult(0,0,driver);
+String replDumpId = getResult(0,1,true,driver);
+LOG.info("Dumped to {} with id {}",replDumpLocn,replDumpId);
+run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", 
driverMirror);
+
+verifyRun("SELECT count(*) from " + dbName + "_dupe.unptned", new 
String[]{"2"}, driverMirror);
+verifyRun("SELECT count(*) from " + dbName + "_dupe.ptned", new 
String[]{"3"}, driverMirror);
+verifyRun("SELECT max(a) from " + dbName + "_dupe.unptned", new 
String[]{"2"}, driverMirror);
+verifyRun("SELECT max(a) from " + dbName + "_dupe.ptned where b=1", new 
String[]{"8"}, driverMirror);
+
+run("CREATE TABLE " + dbName + 

[55/74] [abbrv] hive git commit: HIVE-17225: HoS DPP pruning sink ops can target parallel work objects (Sahil Takiar, reviewed by Sergio Pena)

[weiz]

HIVE-17225: HoS DPP pruning sink ops can target parallel work objects (Sahil 
Takiar, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90e68288
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90e68288
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90e68288

Branch: refs/heads/hive-14535
Commit: 90e682887de28d0c4c5a0caeac8f15d8a8b671af
Parents: e3c71b0
Author: Sahil Takiar 
Authored: Mon Sep 4 15:33:51 2017 -0700
Committer: Sahil Takiar 
Committed: Mon Sep 4 15:33:51 2017 -0700

--
 .../test/resources/testconfiguration.properties |1 +
 .../hadoop/hive/ql/exec/OperatorUtils.java  |   31 +
 .../SparkDynamicPartitionPruningResolver.java   |  146 +++
 .../physical/SparkMapJoinResolver.java  |   35 +-
 .../spark/SparkPartitionPruningSinkDesc.java|   13 +-
 .../spark/SparkSkewJoinProcFactory.java |6 +-
 .../hive/ql/parse/spark/GenSparkUtils.java  |2 +-
 .../hive/ql/parse/spark/SparkCompiler.java  |5 +
 ...ynamic_partition_pruning_recursive_mapjoin.q |  147 +++
 ...spark_vectorized_dynamic_partition_pruning.q |8 +-
 ...ic_partition_pruning_recursive_mapjoin.q.out | 1097 ++
 ...k_vectorized_dynamic_partition_pruning.q.out |  158 +--
 12 files changed, 1540 insertions(+), 109 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/90e68288/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 7385df6..663c95b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1408,6 +1408,7 @@ 
miniSparkOnYarn.only.query.files=spark_combine_equivalent_work.q,\
   spark_dynamic_partition_pruning_3.q,\
   spark_dynamic_partition_pruning_mapjoin_only.q,\
   spark_constprog_dpp.q,\
+  spark_dynamic_partition_pruning_recursive_mapjoin.q,\
   dynamic_rdd_cache.q, \
   spark_multi_insert_parallel_orderby.q,\
   spark_explainuser_1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/90e68288/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
index 7308b5f..e79d100 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
@@ -24,10 +24,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Stack;
 
 import org.apache.hadoop.hive.ql.exec.NodeUtils.Function;
 import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator;
+import org.apache.hadoop.hive.ql.plan.BaseWork;
 import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.slf4j.Logger;
@@ -424,4 +427,32 @@ public class OperatorUtils {
 }
 return true;
   }
+
+  public static Set getOp(BaseWork work, Class clazz) {
+Set ops = new HashSet();
+if (work instanceof MapWork) {
+  Collection opSet = ((MapWork) 
work).getAliasToWork().values();
+  Stack opStack = new Stack();
+  opStack.addAll(opSet);
+
+  while (!opStack.empty()) {
+Operator op = opStack.pop();
+ops.add(op);
+if (op.getChildOperators() != null) {
+  opStack.addAll(op.getChildOperators());
+}
+  }
+} else {
+  ops.addAll(work.getAllOperators());
+}
+
+Set matchingOps =
+  new HashSet();
+for (Operator op : ops) {
+  if (clazz.isInstance(op)) {
+matchingOps.add(op);
+  }
+}
+return matchingOps;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/90e68288/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkDynamicPartitionPruningResolver.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkDynamicPartitionPruningResolver.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkDynamicPartitionPruningResolver.java
new file mode 100644
index 000..4d5c234
--- /dev/null
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkDynamicPartitionPruningResolver.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache 

[53/74] [abbrv] hive git commit: HIVE-17405: HoS DPP ConstantPropagate should use ConstantPropagateOption.SHORTCUT (Sahil Takiar, reviewed by Rui Li)

[weiz]

HIVE-17405: HoS DPP ConstantPropagate should use 
ConstantPropagateOption.SHORTCUT (Sahil Takiar, reviewed by Rui Li)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3c71b00
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3c71b00
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3c71b00

Branch: refs/heads/hive-14535
Commit: e3c71b009bdeb2191f12d60f288486a5d628ecae
Parents: 8f7c578
Author: Sahil Takiar 
Authored: Sat Sep 2 09:38:38 2017 -0700
Committer: Sahil Takiar 
Committed: Sat Sep 2 09:38:38 2017 -0700

--
 .../hive/ql/parse/spark/SparkCompiler.java  |   13 +-
 ...spark_vectorized_dynamic_partition_pruning.q |   35 +-
 .../spark/skewjoin_union_remove_1.q.out |   16 +-
 .../spark/skewjoin_union_remove_2.q.out |6 +-
 .../clientpositive/spark/skewjoinopt1.q.out |   16 +-
 .../clientpositive/spark/skewjoinopt10.q.out|8 +-
 .../clientpositive/spark/skewjoinopt11.q.out|8 +-
 .../clientpositive/spark/skewjoinopt12.q.out|4 +-
 .../clientpositive/spark/skewjoinopt14.q.out|8 +-
 .../clientpositive/spark/skewjoinopt15.q.out|   16 +-
 .../clientpositive/spark/skewjoinopt16.q.out|4 +-
 .../clientpositive/spark/skewjoinopt17.q.out|   12 +-
 .../clientpositive/spark/skewjoinopt19.q.out|8 +-
 .../clientpositive/spark/skewjoinopt2.q.out |   16 +-
 .../clientpositive/spark/skewjoinopt20.q.out|8 +-
 .../clientpositive/spark/skewjoinopt3.q.out |8 +-
 .../clientpositive/spark/skewjoinopt4.q.out |   16 +-
 .../clientpositive/spark/skewjoinopt5.q.out |4 +-
 .../clientpositive/spark/skewjoinopt6.q.out |4 +-
 .../clientpositive/spark/skewjoinopt7.q.out |6 +-
 .../clientpositive/spark/skewjoinopt8.q.out |6 +-
 .../spark/spark_dynamic_partition_pruning.q.out |   12 +-
 ...k_vectorized_dynamic_partition_pruning.q.out | 4280 ++
 23 files changed, 2479 insertions(+), 2035 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e3c71b00/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
index 73e596e..ab8db20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.lib.TypeRule;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx;
 import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
 import org.apache.hadoop.hive.ql.optimizer.SparkRemoveDynamicPruning;
 import 
org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
@@ -130,6 +131,12 @@ public class SparkCompiler extends TaskCompiler {
 // Remove cyclic dependencies for DPP
 runCycleAnalysisForPartitionPruning(procCtx);
 
+// Re-run constant propagation so we fold any new constants introduced by 
the operator optimizers
+// Specifically necessary for DPP because we might have created lots of 
"and true and true" conditions
+if 
(procCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
+  new 
ConstantPropagate(ConstantPropagateProcCtx.ConstantPropagateOption.SHORTCUT).transform(pCtx);
+}
+
 PERF_LOGGER.PerfLogEnd(CLASS_NAME, 
PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE);
   }
 
@@ -284,12 +291,6 @@ public class SparkCompiler extends TaskCompiler {
 List topNodes = new ArrayList();
 topNodes.addAll(parseContext.getTopOps().values());
 ogw.startWalking(topNodes, null);
-
-// need a new run of the constant folding because we might have created 
lots
-// of "and true and true" conditions.
-
if(procCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
-  new ConstantPropagate().transform(parseContext);
-}
   }
 
   private void runSetReducerParallelism(OptimizeSparkProcContext procCtx) 
throws SemanticException {

http://git-wip-us.apache.org/repos/asf/hive/blob/e3c71b00/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
--
diff --git 
a/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
 
b/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
index 8868a34..b4ecbef 100644
--- 

[27/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out 
b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
index 54f5a4e..bb507ce 100644
--- a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
+++ b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
@@ -126,14 +126,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: tables
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: name (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
+  Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -167,21 +167,21 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 125 Data size: 11000 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 125 Data size: 34000 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: string), _col1 (type: string)
   sort order: ++
-  Statistics: Num rows: 125 Data size: 11000 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 125 Data size: 34000 Basic stats: 
COMPLETE Column stats: NONE
 Reducer 3 
 Execution mode: llap
 Reduce Operator Tree:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: string), 
KEY.reducesinkkey1 (type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 125 Data size: 11000 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 125 Data size: 34000 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 125 Data size: 11000 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 125 Data size: 34000 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/join46.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/join46.q.out 
b/ql/src/test/results/clientpositive/llap/join46.q.out
index 079acdd..1f6415c 100644
--- a/ql/src/test/results/clientpositive/llap/join46.q.out
+++ b/ql/src/test/results/clientpositive/llap/join46.q.out
@@ -62,16 +62,16 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: test1
-  Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: int), value (type: int), col_1 
(type: string)
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 6 Data size: 56 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col1 (type: int)
   sort order: +
   Map-reduce partition columns: _col1 (type: int)
-  Statistics: Num rows: 6 Data size: 56 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: int), _col2 (type: 
string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -79,16 +79,16 @@ STAGE PLANS:
 

[40/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out 
b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out
index e4fed11..13eecea 100644
--- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out
@@ -107,19 +107,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 10 Data size: 150 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1950 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 150 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1950 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 10 Data size: 150 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1950 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 10 Data size: 150 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1950 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -127,14 +127,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 84 Data size: 1408 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 84 Data size: 15964 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 84 Data size: 1408 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 80 Data size: 15203 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 84 Data size: 1408 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 80 Data size: 15203 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -144,16 +144,16 @@ STAGE PLANS:
 outputColumnNames: _col0, _col1, _col4
 input vertices:
   0 Map 1
-Statistics: Num rows: 92 Data size: 1548 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 88 Data size: 16723 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: _col0 (type: int), concat(_col1, _col4) 
(type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 92 Data size: 1548 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 88 Data size: 16723 Basic 
stats: COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 92 Data size: 1548 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 88 Data size: 16723 Basic 
stats: COMPLETE Column stats: NONE
 value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -163,10 +163,10 @@ STAGE PLANS:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 
(type: string)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE 
Column stats: NONE
+

[50/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/explain_rearrange.q.out
--
diff --git a/ql/src/test/results/clientpositive/explain_rearrange.q.out 
b/ql/src/test/results/clientpositive/explain_rearrange.q.out
index 2e5f9e4..7d9434b 100644
--- a/ql/src/test/results/clientpositive/explain_rearrange.q.out
+++ b/ql/src/test/results/clientpositive/explain_rearrange.q.out
@@ -135,7 +135,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -190,7 +190,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -225,7 +225,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: 
bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 (type: 
bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -404,7 +404,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -459,7 +459,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -494,7 +494,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: 
bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 (type: 
bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -716,7 +716,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -753,7 +753,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 
(type: bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 
(type: bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -788,7 +788,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: 
bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 (type: 
bigint)
 outputColumnNames: _col0, _col1, _col2
 File Output Operator
   compressed: false
@@ -1011,7 +1011,7 @@ STAGE PLANS:
 1 _col0 (type: int)
   outputColumnNames: _col0, _col1, _col3
   Select Operator
-expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: 
bigint)
+expressions: _col0 (type: int), _col3 (type: bigint), _col1 (type: 
bigint)
 outputColumnNames: _col0, 

[52/74] [abbrv] hive git commit: HIVE-17405: HoS DPP ConstantPropagate should use ConstantPropagateOption.SHORTCUT (Sahil Takiar, reviewed by Rui Li)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/e3c71b00/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
index 699fcc6..f2403aa 100644
--- 
a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
+++ 
b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
@@ -48,7 +48,7 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 2 <- Map 1 (GROUP, 2)
+Reducer 2 <- Map 1 (GROUP, 4)
  A masked pattern was here 
   Vertices:
 Map 1 
@@ -70,6 +70,7 @@ STAGE PLANS:
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
 Statistics: Num rows: 2000 Data size: 21248 Basic 
stats: COMPLETE Column stats: NONE
+Execution mode: vectorized
 Reducer 2 
 Execution mode: vectorized
 Reduce Operator Tree:
@@ -191,11 +192,9 @@ POSTHOOK: Output: database:default
 POSTHOOK: Output: default@srcpart_double_hour
 POSTHOOK: Lineage: srcpart_double_hour.hour SIMPLE 
[(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ]
 POSTHOOK: Lineage: srcpart_double_hour.hr EXPRESSION 
[(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ]
-PREHOOK: query: -- single column, single key
-EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
+PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on 
(srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
 PREHOOK: type: QUERY
-POSTHOOK: query: -- single column, single key
-EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
+POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on 
(srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
@@ -230,16 +229,16 @@ STAGE PLANS:
   outputColumnNames: _col0
   Statistics: Num rows: 1 Data size: 188 Basic stats: 
COMPLETE Column stats: NONE
   Spark Partition Pruning Sink Operator
+Target column: ds (string)
 partition key expr: ds
 Statistics: Num rows: 1 Data size: 188 Basic 
stats: COMPLETE Column stats: NONE
-target column name: ds
 target work: Map 1
 Execution mode: vectorized
 
   Stage: Stage-1
 Spark
   Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL 
SORT, 2)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL 
SORT, 4)
 Reducer 3 <- Reducer 2 (GROUP, 1)
  A masked pattern was here 
   Vertices:
@@ -258,6 +257,7 @@ STAGE PLANS:
   sort order: +
   Map-reduce partition columns: _col0 (type: string)
   Statistics: Num rows: 2000 Data size: 21248 Basic stats: 
COMPLETE Column stats: NONE
+Execution mode: vectorized
 Map 4 
 Map Operator Tree:
 TableScan
@@ -348,7 +348,7 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL 
SORT, 2)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 4 (PARTITION-LEVEL 
SORT, 4)
 Reducer 3 <- Reducer 2 (GROUP, 1)
  A masked pattern was here 
   Vertices:
@@ -367,6 +367,7 @@ STAGE PLANS:
   sort order: +
   Map-reduce partition columns: _col0 (type: string)
   Statistics: Num rows: 2000 Data size: 21248 Basic stats: 
COMPLETE Column stats: NONE
+Execution mode: vectorized
 Map 4 
 Map Operator Tree:
 TableScan
@@ -454,13 +455,9 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcpart
  A masked pattern was here 
 1000
-PREHOOK: query: -- multiple sources, single key
-EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) 
-where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11
+PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on 
(day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = 
'2008-04-08'
 PREHOOK: type: QUERY
-POSTHOOK: query: -- multiple sources, single key
-EXPLAIN 

[42/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out 
b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out
index bac4ae0..57c792f 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out
@@ -121,91 +121,92 @@ STAGE PLANS:
 Tez
  A masked pattern was here 
   Edges:
-Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE)
+Map 2 <- Map 1 (CUSTOM_EDGE)
+Map 3 <- Map 2 (BROADCAST_EDGE)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
+  alias: c
+  Statistics: Num rows: 500 Data size: 1904 Basic stats: 
COMPLETE Column stats: NONE
+  Filter Operator
+predicate: key is not null (type: boolean)
+Statistics: Num rows: 475 Data size: 1808 Basic stats: 
COMPLETE Column stats: NONE
+Select Operator
+  expressions: key (type: int)
+  outputColumnNames: _col0
+  Statistics: Num rows: 475 Data size: 1808 Basic stats: 
COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col0 (type: int)
+sort order: +
+Map-reduce partition columns: _col0 (type: int)
+Statistics: Num rows: 475 Data size: 1808 Basic stats: 
COMPLETE Column stats: NONE
+Execution mode: llap
+LLAP IO: no inputs
+Map 2 
+Map Operator Tree:
+TableScan
   alias: a
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 89488 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (key is not null and value is not null) (type: 
boolean)
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 450 Data size: 80539 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 450 Data size: 80539 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
 keys:
   0 _col0 (type: int)
   1 _col0 (type: int)
-outputColumnNames: _col0, _col1
+outputColumnNames: _col1, _col2
 input vertices:
-  1 Map 2
-Statistics: Num rows: 550 Data size: 10243 Basic 
stats: COMPLETE Column stats: NONE
-Map Join Operator
-  condition map:
-   Inner Join 0 to 1
-  keys:
-0 _col1 (type: string)
-1 _col1 (type: string)
-  outputColumnNames: _col0, _col3
-  input vertices:
-1 Map 3
-  Statistics: Num rows: 605 Data size: 11267 Basic 
stats: COMPLETE Column stats: NONE
-  Select Operator
-expressions: _col0 (type: int), _col3 (type: int)
-outputColumnNames: _col0, _col1
-Statistics: Num rows: 605 Data size: 11267 Basic 
stats: COMPLETE Column stats: NONE
-File Output Operator
-  compressed: false
-  Statistics: Num rows: 605 Data size: 11267 Basic 
stats: COMPLETE Column stats: NONE
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Execution mode: llap
-LLAP IO: no inputs
-Map 2 
-Map Operator Tree:
-TableScan
-  alias: c
-   

[69/74] [abbrv] hive git commit: HIVE-17456: Set current database for external LLAP interface (Jason Dere, reviewed by Sergey Shelukhin)

[weiz]

HIVE-17456: Set current database for external LLAP interface (Jason Dere, 
reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1e3e74e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1e3e74e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1e3e74e5

Branch: refs/heads/hive-14535
Commit: 1e3e74e54d25921f96d1a0e0eae04d4aeb5a40bd
Parents: 5663b97
Author: Jason Dere 
Authored: Thu Sep 7 11:18:19 2017 -0700
Committer: Jason Dere 
Committed: Thu Sep 7 11:18:19 2017 -0700

--
 .../apache/hive/jdbc/TestJdbcWithMiniLlap.java  | 22 +---
 .../hadoop/hive/llap/LlapBaseInputFormat.java   |  5 +
 2 files changed, 24 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1e3e74e5/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
index 68d2ddc..28fa7a5 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
@@ -156,8 +156,17 @@ public class TestJdbcWithMiniLlap {
   }
 
   private void createTestTable(String tableName) throws Exception {
+createTestTable(null, tableName);
+  }
+
+  private void createTestTable(String database, String tableName) throws 
Exception {
 Statement stmt = hs2Conn.createStatement();
 
+if (database != null) {
+  stmt.execute("CREATE DATABASE IF NOT EXISTS " + database);
+  stmt.execute("USE " + database);
+}
+
 // create table
 stmt.execute("DROP TABLE IF EXISTS " + tableName);
 stmt.execute("CREATE TABLE " + tableName
@@ -225,12 +234,12 @@ public class TestJdbcWithMiniLlap {
 
   @Test(timeout = 6)
   public void testNonAsciiStrings() throws Exception {
-createTestTable("testtab1");
+createTestTable("nonascii", "testtab_nonascii");
 
 RowCollector rowCollector = new RowCollector();
 String nonAscii = "À côté du garçon";
-String query = "select value, '" + nonAscii + "' from testtab1 where 
under_col=0";
-int rowCount = processQuery(query, 1, rowCollector);
+String query = "select value, '" + nonAscii + "' from testtab_nonascii 
where under_col=0";
+int rowCount = processQuery("nonascii", query, 1, rowCollector);
 assertEquals(3, rowCount);
 
 assertArrayEquals(new String[] {"val_0", nonAscii}, 
rowCollector.rows.get(0));
@@ -474,6 +483,10 @@ public class TestJdbcWithMiniLlap {
   }
 
   private int processQuery(String query, int numSplits, RowProcessor 
rowProcessor) throws Exception {
+return processQuery(null, query, numSplits, rowProcessor);
+  }
+
+  private int processQuery(String currentDatabase, String query, int 
numSplits, RowProcessor rowProcessor) throws Exception {
 String url = miniHS2.getJdbcURL();
 String user = System.getProperty("user.name");
 String pwd = user;
@@ -488,6 +501,9 @@ public class TestJdbcWithMiniLlap {
 job.set(LlapBaseInputFormat.PWD_KEY, pwd);
 job.set(LlapBaseInputFormat.QUERY_KEY, query);
 job.set(LlapBaseInputFormat.HANDLE_ID, handleId);
+if (currentDatabase != null) {
+  job.set(LlapBaseInputFormat.DB_KEY, currentDatabase);
+}
 
 InputSplit[] splits = inputFormat.getSplits(job, numSplits);
 assertTrue(splits.length > 0);

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3e74e5/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
--
diff --git 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java 
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
index 215f5b1..1708df1 100644
--- 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
+++ 
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
@@ -109,6 +109,7 @@ public class LlapBaseInputFormat>
   public static final String USER_KEY = "llap.if.user";
   public static final String PWD_KEY = "llap.if.pwd";
   public static final String HANDLE_ID = "llap.if.handleid";
+  public static final String DB_KEY = "llap.if.database";
 
   public final String SPLIT_QUERY = "select get_splits(\"%s\",%d)";
   public static final LlapServiceInstance[] serviceInstanceArray = new 
LlapServiceInstance[0];
@@ -210,6 +211,7 @@ public class LlapBaseInputFormat>
 if (query == null) query = job.get(QUERY_KEY);
 if (user == null) user = job.get(USER_KEY);
 if (pwd == 

[45/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out 
b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
index 278c55b..8bd3d12 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
@@ -65,14 +65,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -81,7 +81,7 @@ STAGE PLANS:
   1 _col0 (type: int)
 input vertices:
   1 Map 3
-Statistics: Num rows: 11 Data size: 77 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   mode: hash
@@ -97,19 +97,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -180,14 +180,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 70 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
   Map Join Operator
 condition map:
  Inner Join 0 to 1
@@ -197,18 +197,18 @@ STAGE PLANS:
 outputColumnNames: _col0
 input vertices:
   1 Map 3
-Statistics: Num rows: 11 Data size: 77 Basic stats: 
COMPLETE Column stats: NONE
+

[35/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out 
b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
index 3de40ce..ffc3957 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
@@ -237,33 +237,33 @@ STAGE PLANS:
 TableScan
   alias: srcpart_date
   filterExpr: ((date = '2008-04-08') and ds is not null) 
(type: boolean)
-  Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: ((date = '2008-04-08') and ds is not null) 
(type: boolean)
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: ds (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: _col0 (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   keys: _col0 (type: string)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
   Dynamic Partitioning Event Operator
 Target column: ds (string)
 Target Input: srcpart
 Partition key expr: ds
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic 
stats: COMPLETE Column stats: NONE
 Target Vertex: Map 1
 Execution mode: llap
 LLAP IO: no inputs
@@ -366,19 +366,19 @@ STAGE PLANS:
 TableScan
   alias: srcpart_date
   filterExpr: ((date = '2008-04-08') and ds is not null) 
(type: boolean)
-  Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: ((date = '2008-04-08') and ds is not null) 
(type: boolean)
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: ds (type: string)
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 1 Data size: 21 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2 Data size: 736 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -493,33 +493,33 @@ STAGE PLANS:

[44/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out 
b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
index 5913768..1c6ab8c 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
@@ -63,22 +63,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: clustergroupby
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 92896 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: key
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 500 Data size: 92896 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   keys: key (type: string)
   mode: hash
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 92896 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 500 Data size: 92896 Basic 
stats: COMPLETE Column stats: NONE
 TopN Hash Memory Usage: 0.1
 value expressions: _col1 (type: bigint)
 Execution mode: llap
@@ -91,11 +91,11 @@ STAGE PLANS:
 keys: KEY._col0 (type: string)
 mode: mergepartial
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 250 Data size: 4656 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 250 Data size: 46448 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: string)
   sort order: +
-  Statistics: Num rows: 250 Data size: 4656 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 250 Data size: 46448 Basic stats: 
COMPLETE Column stats: NONE
   TopN Hash Memory Usage: 0.1
   value expressions: _col1 (type: bigint)
 Reducer 3 
@@ -104,13 +104,13 @@ STAGE PLANS:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 
(type: bigint)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 250 Data size: 4656 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 250 Data size: 46448 Basic stats: 
COMPLETE Column stats: NONE
 Limit
   Number of rows: 10
-  Statistics: Num rows: 10 Data size: 180 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 1850 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 10 Data size: 180 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 1850 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -191,22 +191,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: clustergroupby
-  Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 92896 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: key
-Statistics: Num rows: 500 Data size: 9312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 500 Data size: 92896 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   

[31/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out 
b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
index 1dc9ed5..36898ce 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
@@ -130,19 +130,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: over1k
-  Statistics: Num rows: 4443 Data size: 106636 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4443 Data size: 101328 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (t is null or (t = 27)) (type: boolean)
-Statistics: Num rows: 4442 Data size: 106611 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: si (type: smallint), i (type: int), b 
(type: bigint), f (type: float), t (type: tinyint)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4
-  Statistics: Num rows: 4442 Data size: 106611 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col4 (type: tinyint)
 sort order: +
 Map-reduce partition columns: _col4 (type: tinyint)
-Statistics: Num rows: 4442 Data size: 106611 Basic 
stats: COMPLETE Column stats: NONE
+Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: smallint), _col1 
(type: int), _col2 (type: bigint), _col3 (type: float)
 Execution mode: llap
 LLAP IO: no inputs
@@ -152,11 +152,11 @@ STAGE PLANS:
   Select Operator
 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: 
int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: 
tinyint)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-Statistics: Num rows: 4442 Data size: 106611 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
   Dp Sort State: PARTITION_SORTED
-  Statistics: Num rows: 4442 Data size: 106611 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: org.apache.hadoop.mapred.TextInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -205,20 +205,20 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: over1k
-  Statistics: Num rows: 4443 Data size: 106636 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 4443 Data size: 101328 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
 predicate: (t is null or (t = 27)) (type: boolean)
-Statistics: Num rows: 4442 Data size: 106611 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: si (type: smallint), i (type: int), b 
(type: bigint), f (type: float), t (type: tinyint)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4
-  Statistics: Num rows: 4442 Data size: 106611 Basic 
stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 227 Data size: 5177 Basic stats: 
COMPLETE Column stats: NONE
   Limit
 Number of rows: 10
-Statistics: Num rows: 10 Data size: 240 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 220 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 10 Data size: 240 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num 

[61/74] [abbrv] hive git commit: HIVE-17382: Change startsWith relation introduced in HIVE-17316 (Barna Zsombor Klara via Peter Vary)

[weiz]

HIVE-17382: Change startsWith relation introduced in HIVE-17316 (Barna Zsombor 
Klara via Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d55dfab2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d55dfab2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d55dfab2

Branch: refs/heads/hive-14535
Commit: d55dfab2a31d0530b360b1db65bc7c258d5e8d7e
Parents: 21465d1
Author: Peter Vary 
Authored: Wed Sep 6 13:30:54 2017 +0200
Committer: Peter Vary 
Committed: Wed Sep 6 13:30:54 2017 +0200

--
 common/src/java/org/apache/hadoop/hive/conf/HiveConf.java   | 9 +
 .../src/test/org/apache/hadoop/hive/conf/TestHiveConf.java  | 4 
 .../apache/hadoop/hive/conf/TestHiveConfRestrictList.java   | 2 +-
 .../java/org/apache/hive/beeline/QFileBeeLineClient.java| 6 +++---
 .../metastore/datasource/TestDataSourceProviderFactory.java | 6 ++
 .../org/apache/hadoop/hive/ql/session/OperationLog.java | 4 ++--
 .../apache/hive/service/cli/session/HiveSessionImpl.java| 2 +-
 7 files changed, 22 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d55dfab2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6de07d2..cf3f50b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -549,11 +549,11 @@ public class HiveConf extends Configuration {
 "If not set, defaults to the codec extension for text files (e.g. 
\".gz\"), or no extension otherwise."),
 
 HIVE_IN_TEST("hive.in.test", false, "internal usage only, true in test 
mode", true),
-HIVE_IN_TEST_SHORT_LOGS("hive.in.test.short.logs", false,
+HIVE_TESTING_SHORT_LOGS("hive.testing.short.logs", false,
 "internal usage only, used only in test mode. If set true, when 
requesting the " +
 "operation logs the short version (generated by 
LogDivertAppenderForTest) will be " +
 "returned"),
-HIVE_IN_TEST_REMOVE_LOGS("hive.in.test.remove.logs", true,
+HIVE_TESTING_REMOVE_LOGS("hive.testing.remove.logs", true,
 "internal usage only, used only in test mode. If set false, the 
operation logs, and the " +
 "operation log directory will not be removed, so they can be found 
after the test runs."),
 
@@ -3793,7 +3793,8 @@ public class HiveConf extends Configuration {
 + "It is not in list of params that are allowed to be modified at 
runtime");
   }
 }
-if (Iterables.any(restrictList, restrictedVar -> 
restrictedVar.startsWith(name))) {
+if (Iterables.any(restrictList,
+restrictedVar -> name != null && name.startsWith(restrictedVar))) {
   throw new IllegalArgumentException("Cannot modify " + name + " at 
runtime. It is in the list"
   + " of parameters that can't be modified at runtime or is prefixed 
by a restricted variable");
 }
@@ -3809,7 +3810,7 @@ public class HiveConf extends Configuration {
   }
 
   public boolean isHiddenConfig(String name) {
-return hiddenSet.contains(name);
+return Iterables.any(hiddenSet, hiddenVar -> name.startsWith(hiddenVar));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/d55dfab2/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
--
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java 
b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index c914d23..1fac2b0 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -138,6 +138,7 @@ public class TestHiveConf {
 try {
   final String name = HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname;
   conf.verifyAndSet(name, "");
+  conf.verifyAndSet(name + "postfix", "");
   Assert.fail("Setting config property " + name + " should fail");
 } catch (IllegalArgumentException e) {
   // the verifyAndSet in this case is expected to fail with the 
IllegalArgumentException
@@ -147,6 +148,9 @@ public class TestHiveConf {
 conf2.set(HiveConf.ConfVars.METASTOREPWD.varname, "password");
 conf2.set(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, 
"password");
 conf.stripHiddenConfigurations(conf2);
+
Assert.assertTrue(conf.isHiddenConfig(HiveConf.ConfVars.METASTOREPWD.varname + 
"postfix"));
+Assert.assertTrue(
+
conf.isHiddenConfig(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname
 + "postfix"));
 

[02/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/union_remove_26.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out 
b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out
index 9ddc2c8..a8cb207 100644
--- a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out
+++ b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out
@@ -181,10 +181,10 @@ STAGE PLANS:
   aggregations: count(), min(val), max(val)
   mode: hash
   outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint), _col1 (type: 
int), _col2 (type: int)
 Execution mode: llap
 LLAP IO: no inputs
@@ -230,10 +230,10 @@ STAGE PLANS:
 aggregations: count(VALUE._col0), min(VALUE._col1), 
max(VALUE._col2)
 mode: mergepartial
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE 
Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out 
b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
index e63cbf8..0f9abac 100644
--- a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
@@ -125,17 +125,17 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: varchar_udf_1
-  Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: c2 regexp 'val' (type: boolean), c4 regexp 
'val' (type: boolean), (c2 regexp 'val' = c4 regexp 'val') (type: boolean)
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 288 Basic stats: 
COMPLETE Column stats: NONE
 Limit
   Number of rows: 1
-  Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 288 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 1 Data size: 356 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 288 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -203,17 +203,17 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: varchar_udf_1
-  Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: regexp_extract(c2, 'val_([0-9]+)', 1) (type: 
string), regexp_extract(c4, 'val_([0-9]+)', 1) (type: string), 
(regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)) 
(type: 

[51/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by 
Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f7c5788
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f7c5788
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f7c5788

Branch: refs/heads/hive-14535
Commit: 8f7c5788938f3706a42e5ea8350ca6d3500eb15d
Parents: d155565
Author: Vineet Garg 
Authored: Fri Sep 1 22:07:15 2017 -0700
Committer: Vineet Garg 
Committed: Fri Sep 1 22:07:15 2017 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |6 +
 .../test/resources/testconfiguration.properties |1 +
 .../ql/optimizer/calcite/RelOptHiveTable.java   |   93 +-
 .../stats/annotation/StatsRulesProcFactory.java |   12 +-
 .../hadoop/hive/ql/plan/ColStatistics.java  |   12 +
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |  206 +-
 .../clientpositive/join_reordering_no_stats.q   |   63 +
 .../clientpositive/annotate_stats_filter.q.out  |   12 +-
 .../clientpositive/annotate_stats_groupby.q.out |4 +-
 .../clientpositive/annotate_stats_part.q.out|   42 +-
 .../clientpositive/annotate_stats_select.q.out  |4 +-
 .../clientpositive/annotate_stats_table.q.out   |   12 +-
 .../auto_join_reordering_values.q.out   |8 +-
 .../clientpositive/auto_join_stats.q.out|   30 +-
 .../clientpositive/auto_join_stats2.q.out   |   91 +-
 .../clientpositive/auto_sortmerge_join_12.q.out |   12 +-
 .../cbo_rp_annotate_stats_groupby.q.out |4 +-
 .../columnStatsUpdateForStatsOptimizer_2.q.out  |   24 +-
 .../clientpositive/explain_rearrange.q.out  |   24 +-
 ql/src/test/results/clientpositive/join19.q.out |  190 +-
 ql/src/test/results/clientpositive/join42.q.out |  154 +-
 ql/src/test/results/clientpositive/join43.q.out |  136 +-
 .../join_cond_pushdown_unqual1.q.out|  284 +--
 .../join_cond_pushdown_unqual2.q.out|  162 +-
 .../join_cond_pushdown_unqual3.q.out|  284 +--
 .../join_cond_pushdown_unqual4.q.out|  162 +-
 .../results/clientpositive/join_hive_626.q.out  |   60 +-
 .../test/results/clientpositive/join_star.q.out |  124 +-
 .../llap/acid_bucket_pruning.q.out  |8 +-
 .../llap/auto_smb_mapjoin_14.q.out  |  260 +--
 .../llap/auto_sortmerge_join_1.q.out|   48 +-
 .../llap/auto_sortmerge_join_10.q.out   |   42 +-
 .../llap/auto_sortmerge_join_11.q.out   |   64 +-
 .../llap/auto_sortmerge_join_12.q.out   |   26 +-
 .../llap/auto_sortmerge_join_13.q.out   |  134 +-
 .../llap/auto_sortmerge_join_14.q.out   |   24 +-
 .../llap/auto_sortmerge_join_15.q.out   |   24 +-
 .../llap/auto_sortmerge_join_2.q.out|   32 +-
 .../llap/auto_sortmerge_join_3.q.out|   48 +-
 .../llap/auto_sortmerge_join_4.q.out|   48 +-
 .../llap/auto_sortmerge_join_5.q.out|   44 +-
 .../llap/auto_sortmerge_join_6.q.out|  682 ---
 .../llap/auto_sortmerge_join_7.q.out|   48 +-
 .../llap/auto_sortmerge_join_8.q.out|   48 +-
 .../llap/auto_sortmerge_join_9.q.out|  488 ++---
 .../clientpositive/llap/bucket_groupby.q.out|  198 +-
 .../llap/bucket_map_join_tez1.q.out | 1121 ++-
 .../llap/bucket_map_join_tez2.q.out |  322 +--
 .../clientpositive/llap/bucketmapjoin1.q.out|   88 +-
 .../clientpositive/llap/bucketmapjoin2.q.out|   66 +-
 .../clientpositive/llap/bucketmapjoin3.q.out|   44 +-
 .../clientpositive/llap/bucketmapjoin4.q.out|   44 +-
 .../clientpositive/llap/bucketmapjoin7.q.out|   24 +-
 .../clientpositive/llap/bucketpruning1.q.out|  208 +-
 .../llap/bucketsortoptimize_insert_2.q.out  |  144 +-
 .../llap/bucketsortoptimize_insert_6.q.out  |  168 +-
 .../llap/bucketsortoptimize_insert_7.q.out  |   72 +-
 .../columnStatsUpdateForStatsOptimizer_1.q.out  |   24 +-
 .../llap/column_access_stats.q.out  |  138 +-
 .../llap/column_table_stats.q.out   |   24 +-
 .../llap/column_table_stats_orc.q.out   |   20 +-
 .../clientpositive/llap/constprog_dpp.q.out |   20 +-
 .../llap/constprog_semijoin.q.out   |  150 +-
 .../llap/correlationoptimizer4.q.out|  324 +--
 .../results/clientpositive/llap/count.q.out |  144 +-
 .../llap/cross_product_check_1.q.out|  150 +-
 .../llap/cross_product_check_2.q.out|  132 +-
 .../results/clientpositive/llap/cte_3.q.out |8 +-
 .../results/clientpositive/llap/cte_5.q.out |   10 +-
 .../results/clientpositive/llap/cte_mat_3.q.out |   18 +-
 .../results/clientpositive/llap/cte_mat_4.q.out |   36 +-
 .../results/clientpositive/llap/cte_mat_5.q.out |   16 +-
 .../llap/disable_merge_for_bucketing.q.out 

[62/74] [abbrv] hive git commit: HIVE-17389: Yetus is always failing on rat checks (Barna Zsombor Klara via Peter Vary)

[weiz]

HIVE-17389: Yetus is always failing on rat checks (Barna Zsombor Klara via 
Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1923e21b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1923e21b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1923e21b

Branch: refs/heads/hive-14535
Commit: 1923e21bc158c58ac3c1bd83ea3132fe6ba94ceb
Parents: d55dfab
Author: Peter Vary 
Authored: Wed Sep 6 13:33:05 2017 +0200
Committer: Peter Vary 
Committed: Wed Sep 6 13:33:05 2017 +0200

--
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1923e21b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9ec121a..bf28a96 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1230,6 +1230,8 @@
**/*.html
**/sit
   **/test/queries/**/*.sql
+**/patchprocess/**
+**/metastore_db/**
  

   

[73/74] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/test/results/clientpositive/mm_all.q.out
--
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index ea60414,000..b69ac00
mode 100644,00..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2147 -1,0 +1,2116 @@@
 +PREHOOK: query: drop table intermediate
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate(key int) partitioned by (p int) 
stored as orc
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate
 +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) 
stored as orc
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate
 +PREHOOK: query: insert into table intermediate partition(p='455') select 
distinct key from src where key >= 0 order by key desc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=455
 +POSTHOOK: query: insert into table intermediate partition(p='455') select 
distinct key from src where key >= 0 order by key desc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=455
 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='456') select 
distinct key from src where key is not null order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=456
 +POSTHOOK: query: insert into table intermediate partition(p='456') select 
distinct key from src where key is not null order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=456
 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='457') select 
distinct key from src where key >= 100 order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=457
 +POSTHOOK: query: insert into table intermediate partition(p='457') select 
distinct key from src where key >= 100 order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=457
 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) 
stored as orc tblproperties ("transactional"="true", 
"transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) 
stored as orc tblproperties ("transactional"="true", 
"transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: explain insert into table part_mm partition(key_mm=455) 
select key from intermediate
 +PREHOOK: type: QUERY
 +POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) 
select key from intermediate
 +POSTHOOK: type: QUERY
 +STAGE DEPENDENCIES:
 +  Stage-1 is a root stage
 +  Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
 +  Stage-4
 +  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
 +  Stage-2 depends on stages: Stage-0
 +  Stage-3
 +  Stage-5
 +  Stage-6 depends on stages: Stage-5
 +
 +STAGE PLANS:
 +  Stage: Stage-1
 +Map Reduce
 +  Map Operator Tree:
 +  TableScan
 +alias: intermediate
 +Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +Select Operator
 +  expressions: key (type: int)
 +  outputColumnNames: _col0
 +  Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +  File Output Operator
 +compressed: false
 +Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +table:
 +input format: 
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +output format: 
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +name: default.part_mm
 +

[58/74] [abbrv] hive git commit: errata.txt - commit message for HIVE-17420 referred incorrectly to HIVE-17426

[weiz]

errata.txt - commit message for HIVE-17420 referred incorrectly to HIVE-17426


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a09b78b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a09b78b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a09b78b

Branch: refs/heads/hive-14535
Commit: 9a09b78b6067985b24bafb70e1c074653d927d6c
Parents: eb0034c
Author: Thejas M Nair 
Authored: Tue Sep 5 09:11:15 2017 -0700
Committer: Thejas M Nair 
Committed: Tue Sep 5 09:11:15 2017 -0700

--
 errata.txt | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9a09b78b/errata.txt
--
diff --git a/errata.txt b/errata.txt
index d3f96bf..87e41b8 100644
--- a/errata.txt
+++ b/errata.txt
@@ -1,6 +1,7 @@
 Commits with the wrong or no JIRA referenced:
 
 git commit   branch jira   url
+eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a master HIVE-17420 
https://issues.apache.org/jira/browse/HIVE-17420
 f1aae85f197de09d4b86143f7f13d5aa21d2eb85 master HIVE-16431 
https://issues.apache.org/jira/browse/HIVE-16431
 cbab5b29f26ceb3d4633ade9647ce8bcb2f020a0 master HIVE-16422 
https://issues.apache.org/jira/browse/HIVE-16422
 e6143de2b0c3f53d32db8a743119e3a8080d4f85 master HIVE-16425 
https://issues.apache.org/jira/browse/HIVE-16425

[04/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out 
b/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out
index 67cd110..ce871c4 100644
--- a/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out
@@ -160,25 +160,25 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: x
-  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: PARTIAL
+  Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE 
Column stats: PARTIAL
   Filter Operator
 predicate: ((date < '2014-09-02') and (u <> 0)) (type: 
boolean)
-Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
 Select Operator
   expressions: u (type: bigint), date (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
   Group By Operator
 aggregations: min(_col1)
 keys: _col0 (type: bigint)
 mode: hash
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
 Reduce Output Operator
   key expressions: _col0 (type: bigint)
   sort order: +
   Map-reduce partition columns: _col0 (type: bigint)
-  Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
   value expressions: _col1 (type: string)
 Execution mode: llap
 LLAP IO: unknown
@@ -186,40 +186,40 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: v
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: t is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: t (type: string), st (type: string)
   sort order: ++
   Map-reduce partition columns: t (type: string), st 
(type: string)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: all inputs
 Map 5 
 Map Operator Tree:
 TableScan
   alias: y
-  Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE 
Column stats: PARTIAL
+  Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE 
Column stats: PARTIAL
   Filter Operator
 predicate: ((date < '2014-09-02') and (u <> 0)) (type: 
boolean)
-Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
 Select Operator
   expressions: u (type: bigint), date (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 184 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 1 Data size: 192 Basic stats: 
COMPLETE Column stats: PARTIAL
   Group By Operator
 aggregations: min(_col1)
 keys: _col0 (type: bigint)
 mode: hash
 outputColumnNames: _col0, _col1
-   

[16/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out 
b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
index 2fb3fe8..a76d64b 100644
--- 
a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
+++ 
b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
@@ -79,7 +79,7 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: table_add_int_permute_select
-  Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   TableScan Vectorization:
   native: true
   projectedOutputColumns: [0, 1, 2, 3]
@@ -90,13 +90,13 @@ STAGE PLANS:
 className: VectorSelectOperator
 native: true
 projectedOutputColumns: [0, 1, 2]
-Statistics: Num rows: 6 Data size: 120 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
   File Sink Vectorization:
   className: VectorFileSinkOperator
   native: false
-  Statistics: Num rows: 6 Data size: 120 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -239,7 +239,7 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: table_add_int_string_permute_select
-  Statistics: Num rows: 6 Data size: 127 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   TableScan Vectorization:
   native: true
   projectedOutputColumns: [0, 1, 2, 3, 4]
@@ -250,13 +250,13 @@ STAGE PLANS:
 className: VectorSelectOperator
 native: true
 projectedOutputColumns: [0, 1, 2]
-Statistics: Num rows: 6 Data size: 127 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
   File Sink Vectorization:
   className: VectorFileSinkOperator
   native: false
-  Statistics: Num rows: 6 Data size: 127 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1152 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -461,7 +461,7 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: table_change_string_group_double
-  Statistics: Num rows: 6 Data size: 482 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1272 Basic stats: 
COMPLETE Column stats: NONE
   TableScan Vectorization:
   native: true
   projectedOutputColumns: [0, 1, 2, 3, 4]
@@ -472,13 +472,13 @@ STAGE PLANS:
 className: VectorSelectOperator
 native: true
 projectedOutputColumns: [0, 1, 2, 3, 4]
-Statistics: Num rows: 6 Data size: 482 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 6 Data size: 1272 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
   File Sink Vectorization:
   className: VectorFileSinkOperator
   native: false
-  Statistics: Num rows: 6 Data size: 482 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 6 Data size: 1272 Basic stats: 
COMPLETE Column 

[17/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out
 
b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out
index 757ea3a..719d802 100644
--- 
a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out
+++ 
b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out
@@ -266,14 +266,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_change_various_various_boolean_to_bigint
-  Statistics: Num rows: 10 Data size: 5126 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 10 Data size: 9566 Basic stats: 
COMPLETE Column stats: PARTIAL
   Select Operator
 expressions: insert_num (type: int), part (type: int), c1 
(type: boolean), c2 (type: boolean), c3 (type: boolean), c4 (type: boolean), c5 
(type: boolean), c6 (type: boolean), c7 (type: boolean), c8 (type: boolean), c9 
(type: boolean), c10 (type: tinyint), c11 (type: tinyint), c12 (type: tinyint), 
c13 (type: tinyint), c14 (type: tinyint), c15 (type: tinyint), c16 (type: 
tinyint), c17 (type: tinyint), c18 (type: tinyint), c19 (type: tinyint), c20 
(type: tinyint), c21 (type: smallint), c22 (type: smallint), c23 (type: 
smallint), c24 (type: smallint), c25 (type: smallint), c26 (type: smallint), 
c27 (type: smallint), c28 (type: smallint), c29 (type: smallint), c30 (type: 
smallint), c31 (type: smallint), c32 (type: int), c33 (type: int), c34 (type: 
int), c35 (type: int), c36 (type: int), c37 (type: int), c38 (type: int), c39 
(type: int), c40 (type: int), c41 (type: int), c42 (type: int), c43 (type: 
bigint), c44 (type: bigint), c45 (type: bigint), c46 (type: bigint), c4
 7 (type: bigint), c48 (type: bigint), c49 (type: bigint), c50 (type: bigint), 
c51 (type: bigint), c52 (type: bigint), c53 (type: bigint), b (type: string)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, 
_col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, 
_col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, 
_col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, 
_col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, 
_col55
-Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 10 Data size: 4480 Basic stats: 
COMPLETE Column stats: PARTIAL
 File Output Operator
   compressed: false
-  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 10 Data size: 4480 Basic stats: 
COMPLETE Column stats: PARTIAL
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -491,14 +491,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_change_various_various_decimal_to_double
-  Statistics: Num rows: 6 Data size: 2735 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 6 Data size: 12047 Basic stats: 
COMPLETE Column stats: PARTIAL
   Select Operator
 expressions: insert_num (type: int), part (type: int), c1 
(type: decimal(38,18)), c2 (type: decimal(38,18)), c3 (type: decimal(38,18)), 
c4 (type: decimal(38,18)), c5 (type: decimal(38,18)), c6 (type: 
decimal(38,18)), c7 (type: decimal(38,18)), c8 (type: decimal(38,18)), c9 
(type: decimal(38,18)), c10 (type: decimal(38,18)), c11 (type: decimal(38,18)), 
c12 (type: float), c13 (type: float), c14 (type: float), c15 (type: float), c16 
(type: float), c17 (type: float), c18 (type: float), c19 (type: float), c20 
(type: float), c21 (type: float), c22 (type: float), c23 (type: double), c24 
(type: double), c25 (type: double), c26 (type: double), c27 (type: double), c28 
(type: double), c29 (type: double), c30 (type: double), c31 (type: double), c32 
(type: double), c33 (type: double), b (type: string)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, 
_col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, 
_col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, 
_col35
- 

[20/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out 
b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
index 8c6f97e..eefc091 100644
--- a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
@@ -133,19 +133,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: orc_pred
-  Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1049 Data size: 3992 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: hash(t) (type: int)
 outputColumnNames: _col0
-Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1049 Data size: 3992 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: sum(_col0)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint)
 Execution mode: llap
 LLAP IO: all inputs
@@ -156,10 +156,10 @@ STAGE PLANS:
 aggregations: sum(VALUE._col0)
 mode: mergepartial
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE 
Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -191,19 +191,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: orc_pred
-  Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1049 Data size: 3992 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: hash(t) (type: int)
 outputColumnNames: _col0
-Statistics: Num rows: 1049 Data size: 311170 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1049 Data size: 3992 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: sum(_col0)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 12 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint)
 Execution mode: llap
 LLAP IO: all inputs
@@ -214,10 +214,10 @@ STAGE PLANS:
 aggregations: sum(VALUE._col0)
 mode: mergepartial
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE 
Column stats: NONE
   table:

[14/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/skewjoin.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/skewjoin.q.out 
b/ql/src/test/results/clientpositive/llap/skewjoin.q.out
index 794ef75..5a87f93 100644
--- a/ql/src/test/results/clientpositive/llap/skewjoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/skewjoin.q.out
@@ -220,15 +220,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: string)
   sort order: +
   Map-reduce partition columns: key (type: string)
-  Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: val (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -236,15 +236,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: string)
   sort order: +
   Map-reduce partition columns: key (type: string)
-  Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: val (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -252,15 +252,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: c
-  Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: string)
   sort order: +
   Map-reduce partition columns: key (type: string)
-  Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: val (type: string)
 Execution mode: llap
 LLAP IO: no inputs
@@ -268,15 +268,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: d
-  Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: key (type: string)
   sort order: +
   Map-reduce partition columns: key (type: string)
-  Statistics: Num rows: 1 Data size: 30 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 368 Basic stats: 
COMPLETE Column 

[01/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

Repository: hive
Updated Branches:
  refs/heads/hive-14535 42a38577b -> 18d133857


http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out 
b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
index b4386c8..6997af9 100644
--- a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
@@ -68,32 +68,32 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
   Merge Join Operator
 condition map:
  Inner Join 0 to 1
 keys:
   0 _col0 (type: int)
   1 _col0 (type: int)
-Statistics: Num rows: 11 Data size: 1023 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 11 Data size: 44 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: count()
   Group By Vectorization:
@@ -206,25 +206,25 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 10 Data size: 40 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int)
   outputColumnNames: _col0
-  Statistics: Num rows: 10 Data size: 930 Basic stats: 
COMPLETE Column stats: NONE
+  

[13/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out 
b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out
index 6c078ad..1023641 100644
--- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out
+++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out
@@ -55,22 +55,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: a
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 89488 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 85013 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 475 Data size: 85013 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 null sort order: a
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 85013 Basic 
stats: COMPLETE Column stats: NONE
 tag: 0
 value expressions: _col1 (type: string)
 auto parallelism: true
@@ -135,22 +135,22 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: b
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 500 Data size: 89488 Basic stats: 
COMPLETE Column stats: NONE
   GatherStats: false
   Filter Operator
 isSamplingPred: false
 predicate: key is not null (type: boolean)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 85013 Basic stats: 
COMPLETE Column stats: NONE
 Select Operator
   expressions: key (type: int), value (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 475 Data size: 85013 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 null sort order: a
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 500 Data size: 5312 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 475 Data size: 85013 Basic 
stats: COMPLETE Column stats: NONE
 tag: 1
 value expressions: _col1 (type: string)
 auto parallelism: true
@@ -223,12 +223,12 @@ STAGE PLANS:
   1 _col0 (type: int)
 outputColumnNames: _col0, _col1, _col2, _col3
 Position of Big Table: 0
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 522 Data size: 93514 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   null sort order: a
   sort order: +
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 522 Data size: 93514 Basic stats: 
COMPLETE Column stats: NONE
   tag: -1
   TopN: 10
   TopN Hash Memory Usage: 0.1
@@ -241,16 +241,16 @@ STAGE PLANS:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 
(type: string), VALUE._col1 (type: int), VALUE._col2 

[07/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out 
b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index 40c6c82..ba5ccad 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -112,19 +112,19 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: part_null
-  Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 814 Data size: 3100 Basic stats: 
COMPLETE Column stats: NONE
   Select Operator
 expressions: p_size (type: int)
 outputColumnNames: p_size
-Statistics: Num rows: 814 Data size: 3256 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 814 Data size: 3100 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   aggregations: avg(p_size)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 1 Data size: 76 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 80 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 76 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 80 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: 
struct)
 Execution mode: llap
 LLAP IO: no inputs
@@ -139,14 +139,14 @@ STAGE PLANS:
   1 
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9
 residual filter predicates: {(UDFToDouble(_col5) > _col9)}
-Statistics: Num rows: 8 Data size: 5568 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 8 Data size: 5600 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: int), _col1 (type: string), _col2 
(type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), 
_col6 (type: string), _col7 (type: double), _col8 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8
-  Statistics: Num rows: 8 Data size: 5568 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 8 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 8 Data size: 5568 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 8 Data size: 5600 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -158,10 +158,10 @@ STAGE PLANS:
 aggregations: avg(VALUE._col0)
 mode: mergepartial
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col0 (type: double)
 
   Stage: Stage-0
@@ -258,14 +258,14 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: tempty
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: c (type: char(2))
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 86 Basic stats: 
COMPLETE Column stats: NONE
 Reduce Output Operator
   sort order: 
-  Statistics: Num rows: 

[06/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/subquery_select.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out 
b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
index 20e8ed1..0aaed97 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
@@ -2178,30 +2178,30 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: tnull
-  Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: i (type: int)
 outputColumnNames: i
-Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 Group By Operator
   aggregations: count(), count(i)
   mode: hash
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 sort order: 
-Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 1 Data size: 20 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint), _col1 (type: 
bigint)
   Group By Operator
 keys: i (type: int)
 mode: hash
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 1 Data size: 2 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
 LLAP IO: no inputs
 Reducer 2 
@@ -2214,12 +2214,12 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 26 Data size: 546 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 26 Data size: 650 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: int)
   sort order: +
   Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 26 Data size: 546 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 26 Data size: 650 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col1 (type: bigint), _col2 (type: bigint)
 Reducer 3 
 Execution mode: llap
@@ -2231,14 +2231,14 @@ STAGE PLANS:
   0 _col0 (type: int)
   1 _col0 (type: int)
 outputColumnNames: _col0, _col1, _col2, _col4
-Statistics: Num rows: 28 Data size: 600 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 28 Data size: 715 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: int), CASE WHEN ((_col1 = 0)) THEN 
(false) WHEN (_col4 is not null) THEN (true) WHEN (_col0 is null) THEN (null) 
WHEN ((_col2 < _col1)) THEN (null) ELSE (false) END (type: boolean)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 28 Data size: 600 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 28 Data size: 715 Basic stats: 
COMPLETE Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 28 Data size: 600 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 28 Data size: 715 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 

[10/74] [abbrv] [partial] hive git commit: HIVE-16811 : Estimate statistics in absence of stats (Vineet Garg, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/8f7c5788/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/sqlmerge.q.out 
b/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
index 562d444..61349f8 100644
--- a/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
+++ b/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
@@ -52,12 +52,12 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: t
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Output Operator
 key expressions: a (type: int)
 sort order: +
 Map-reduce partition columns: a (type: int)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 value expressions: ROW__ID (type: 
struct)
 Execution mode: llap
 LLAP IO: may be used (ACID table)
@@ -65,12 +65,12 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: s
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Output Operator
 key expressions: a (type: int)
 sort order: +
 Map-reduce partition columns: a (type: int)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE 
Column stats: NONE
 value expressions: b (type: int)
 Execution mode: llap
 LLAP IO: all inputs
@@ -84,62 +84,62 @@ STAGE PLANS:
   0 a (type: int)
   1 a (type: int)
 outputColumnNames: _col0, _col4, _col5, _col6
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 Filter Operator
   predicate: ((_col5 > 8) and (_col0 = _col5)) (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: _col4 (type: 
struct)
 outputColumnNames: _col0
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: 
struct)
   sort order: +
   Map-reduce partition columns: UDFToInteger(_col0) (type: 
int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: 
COMPLETE Column stats: NONE
 Filter Operator
   predicate: ((_col5 <= 8) and (_col0 = _col5)) (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: _col4 (type: 
struct), _col0 (type: int)
 outputColumnNames: _col0, _col1
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
 Reduce Output Operator
   key expressions: _col0 (type: 
struct)
   sort order: +
   Map-reduce partition columns: UDFToInteger(_col0) (type: 
int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: 
PARTIAL Column stats: NONE
+  Statistics: Num rows: 1 Data size: 4 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col1 (type: int)
 

[38/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
--
diff --cc 
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
index 000,6cc7157..3759348
mode 00,100644..100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
@@@ -1,0 -1,256 +1,256 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.hadoop.hive.metastore.model;
+ 
+ import java.util.List;
+ import java.util.Map;
+ 
+ public class MTable {
+   
+   private String tableName;
+   private MDatabase database;
+   private MStorageDescriptor sd;
+   private String owner;
+   private int createTime;
+   private int lastAccessTime;
+   private int retention;
+   private List partitionKeys;
+   private Map parameters;
+   private String viewOriginalText;
+   private String viewExpandedText;
+   private boolean rewriteEnabled;
+   private String tableType;
+ 
+   public MTable() {}
+ 
+   /**
+* @param tableName
+* @param database
+* @param sd
+* @param owner
+* @param createTime
+* @param lastAccessTime
+* @param retention
+* @param partitionKeys
+* @param parameters
+* @param viewOriginalText
+* @param viewExpandedText
+* @param tableType
+*/
+   public MTable(String tableName, MDatabase database, MStorageDescriptor sd, 
String owner,
+   int createTime, int lastAccessTime, int retention, List 
partitionKeys,
 -  Map parameters, String viewOriginalText, String 
viewExpandedText,
 -  boolean rewriteEnabled, String tableType) {
++  Map parameters,
++  String viewOriginalText, String viewExpandedText, boolean 
rewriteEnabled, String tableType) {
+ this.tableName = tableName;
+ this.database = database;
+ this.sd = sd;
+ this.owner = owner;
+ this.createTime = createTime;
+ this.setLastAccessTime(lastAccessTime);
+ this.retention = retention;
+ this.partitionKeys = partitionKeys;
+ this.parameters = parameters;
+ this.viewOriginalText = viewOriginalText;
+ this.viewExpandedText = viewExpandedText;
+ this.rewriteEnabled = rewriteEnabled;
+ this.tableType = tableType;
+   }
+ 
+   /**
+* @return the tableName
+*/
+   public String getTableName() {
+ return tableName;
+   }
+ 
+   /**
+* @param tableName the tableName to set
+*/
+   public void setTableName(String tableName) {
+ this.tableName = tableName;
+   }
+ 
+   /**
+* @return the sd
+*/
+   public MStorageDescriptor getSd() {
+ return sd;
+   }
+ 
+   /**
+* @param sd the sd to set
+*/
+   public void setSd(MStorageDescriptor sd) {
+ this.sd = sd;
+   }
+ 
+   /**
+* @return the partKeys
+*/
+   public List getPartitionKeys() {
+ return partitionKeys;
+   }
+ 
+   /**
+* @param partKeys the partKeys to set
+*/
+   public void setPartitionKeys(List partKeys) {
+ this.partitionKeys = partKeys;
+   }
+ 
+   /**
+* @return the parameters
+*/
+   public Map getParameters() {
+ return parameters;
+   }
+ 
+   /**
+* @param parameters the parameters to set
+*/
+   public void setParameters(Map parameters) {
+ this.parameters = parameters;
+   }
+ 
+   /**
+* @return the original view text, or null if this table is not a view
+*/
+   public String getViewOriginalText() {
+ return viewOriginalText;
+   }
+ 
+   /**
+* @param viewOriginalText the original view text to set
+*/
+   public void setViewOriginalText(String viewOriginalText) {
+ this.viewOriginalText = viewOriginalText;
+   }
+ 
+   /**
+* @return the expanded view text, or null if this table is not a view
+*/
+   public String getViewExpandedText() {
+ return viewExpandedText;
+   }
+ 
+   /**
+* @param viewExpandedText the expanded view text to set
+*/
+   public void 

[03/50] [abbrv] hive git commit: HIVE-17341 - DbTxnManger.startHeartbeat() - randomize initial delay (Eugene Koifman, reviewed by Wei Zheng)

[weiz]

HIVE-17341 - DbTxnManger.startHeartbeat() - randomize initial delay (Eugene 
Koifman, reviewed by Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1938eeb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1938eeb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1938eeb5

Branch: refs/heads/hive-14535
Commit: 1938eeb5948bd1e2443d785cf4b1896cc2c3d4c5
Parents: 733bc5f
Author: Eugene Koifman 
Authored: Fri Aug 25 12:47:01 2017 -0700
Committer: Eugene Koifman 
Committed: Fri Aug 25 12:47:01 2017 -0700

--
 .../java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java   | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1938eeb5/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 03d5b09..5dec791 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -639,8 +639,12 @@ public final class DbTxnManager extends HiveTxnManagerImpl 
{
 if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && 
conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER)) {
   initialDelay = 0;
 } else if (initialDelay == 0) {
-  initialDelay = heartbeatInterval;
+  /*make initialDelay a random number in [0, 0.75*heartbeatInterval] so 
that if a lot
+  of queries land on the server at the same time and all get blocked on 
lack of
+  resources, that they all don't start heartbeating at the same time*/
+  initialDelay = (long)Math.floor(heartbeatInterval * 0.75 * 
Math.random());
 }
+
 heartbeatTask = heartbeatExecutorService.scheduleAtFixedRate(
 heartbeater, initialDelay, heartbeatInterval, TimeUnit.MILLISECONDS);
 LOG.info("Started heartbeat with delay/interval = " + initialDelay + "/" + 
heartbeatInterval +

[22/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h 
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
index 8406b42..3094394 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
@@ -173,6 +173,7 @@ class ThriftHiveMetastoreIf : virtual public  
::facebook::fb303::FacebookService
   virtual void add_dynamic_partitions(const AddDynamicPartitions& rqst) = 0;
   virtual void get_next_notification(NotificationEventResponse& _return, const 
NotificationEventRequest& rqst) = 0;
   virtual void get_current_notificationEventId(CurrentNotificationEventId& 
_return) = 0;
+  virtual void get_notification_events_count(NotificationEventsCountResponse& 
_return, const NotificationEventsCountRequest& rqst) = 0;
   virtual void fire_listener_event(FireEventResponse& _return, const 
FireEventRequest& rqst) = 0;
   virtual void flushCache() = 0;
   virtual void cm_recycle(CmRecycleResponse& _return, const CmRecycleRequest& 
request) = 0;
@@ -692,6 +693,9 @@ class ThriftHiveMetastoreNull : virtual public 
ThriftHiveMetastoreIf , virtual p
   void get_current_notificationEventId(CurrentNotificationEventId& /* _return 
*/) {
 return;
   }
+  void get_notification_events_count(NotificationEventsCountResponse& /* 
_return */, const NotificationEventsCountRequest& /* rqst */) {
+return;
+  }
   void fire_listener_event(FireEventResponse& /* _return */, const 
FireEventRequest& /* rqst */) {
 return;
   }
@@ -19527,6 +19531,110 @@ class 
ThriftHiveMetastore_get_current_notificationEventId_presult {
 
 };
 
+typedef struct _ThriftHiveMetastore_get_notification_events_count_args__isset {
+  _ThriftHiveMetastore_get_notification_events_count_args__isset() : 
rqst(false) {}
+  bool rqst :1;
+} _ThriftHiveMetastore_get_notification_events_count_args__isset;
+
+class ThriftHiveMetastore_get_notification_events_count_args {
+ public:
+
+  ThriftHiveMetastore_get_notification_events_count_args(const 
ThriftHiveMetastore_get_notification_events_count_args&);
+  ThriftHiveMetastore_get_notification_events_count_args& operator=(const 
ThriftHiveMetastore_get_notification_events_count_args&);
+  ThriftHiveMetastore_get_notification_events_count_args() {
+  }
+
+  virtual ~ThriftHiveMetastore_get_notification_events_count_args() throw();
+  NotificationEventsCountRequest rqst;
+
+  _ThriftHiveMetastore_get_notification_events_count_args__isset __isset;
+
+  void __set_rqst(const NotificationEventsCountRequest& val);
+
+  bool operator == (const 
ThriftHiveMetastore_get_notification_events_count_args & rhs) const
+  {
+if (!(rqst == rhs.rqst))
+  return false;
+return true;
+  }
+  bool operator != (const 
ThriftHiveMetastore_get_notification_events_count_args ) const {
+return !(*this == rhs);
+  }
+
+  bool operator < (const 
ThriftHiveMetastore_get_notification_events_count_args & ) const;
+
+  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+
+class ThriftHiveMetastore_get_notification_events_count_pargs {
+ public:
+
+
+  virtual ~ThriftHiveMetastore_get_notification_events_count_pargs() throw();
+  const NotificationEventsCountRequest* rqst;
+
+  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct 
_ThriftHiveMetastore_get_notification_events_count_result__isset {
+  _ThriftHiveMetastore_get_notification_events_count_result__isset() : 
success(false) {}
+  bool success :1;
+} _ThriftHiveMetastore_get_notification_events_count_result__isset;
+
+class ThriftHiveMetastore_get_notification_events_count_result {
+ public:
+
+  ThriftHiveMetastore_get_notification_events_count_result(const 
ThriftHiveMetastore_get_notification_events_count_result&);
+  ThriftHiveMetastore_get_notification_events_count_result& operator=(const 
ThriftHiveMetastore_get_notification_events_count_result&);
+  ThriftHiveMetastore_get_notification_events_count_result() {
+  }
+
+  virtual ~ThriftHiveMetastore_get_notification_events_count_result() throw();
+  NotificationEventsCountResponse success;
+
+  _ThriftHiveMetastore_get_notification_events_count_result__isset __isset;
+
+  void __set_success(const NotificationEventsCountResponse& val);
+
+  bool operator == (const 
ThriftHiveMetastore_get_notification_events_count_result & rhs) const
+  {
+if (!(success == rhs.success))
+  return false;
+return true;
+  }
+  bool operator != (const 
ThriftHiveMetastore_get_notification_events_count_result ) const {
+return !(*this == rhs);
+  }
+
+  bool operator < (const 
ThriftHiveMetastore_get_notification_events_count_result & ) const;
+
+  uint32_t 

[14/50] [abbrv] hive git commit: HIVE-17375: stddev_samp, var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_12.q.out
--
diff --git a/ql/src/test/results/clientpositive/vectorization_12.q.out 
b/ql/src/test/results/clientpositive/vectorization_12.q.out
index df3f047..63ea984 100644
--- a/ql/src/test/results/clientpositive/vectorization_12.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_12.q.out
@@ -284,535 +284,535 @@ ORDER BY ctimestamp1, cdouble, cbigint, cstring1
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@alltypesorc
  A masked pattern was here 
--1645852809false   DUSKf88aNULL6764.0  -4.3506048E7
1645852809  1   -1645852809 0.0 6764.0  -6764.0 6764.0  
4.3506048E7 -1645858447.15  -1645852809 -1.554726368159204E-4   6764.0  
6764.0  4.3512812E7 0.0
--1645852809false   G7Ve8Px6a7J0DafBodF8JMmaNULL-1291.0 
8303712.0   1645852809  1   -1645852809 0.0 -1291.0 1291.0  
-1291.0 -8303712.0  -1645858447.15  -1645852809 -1.554726368159204E-4   
-1291.0 -1291.0 -8305003.0  0.0
--1645852809false   K7tGy146ydkaNULL-1236.0 7949952.0   
1645852809  1   -1645852809 0.0 -1236.0 1236.0  -1236.0 
-7949952.0  -1645858447.15  -1645852809 -1.554726368159204E-4   -1236.0 
-1236.0 -7951188.0  0.0
--1645852809false   OHG2wWD83Ba NULL6914.0  -4.4470848E7
1645852809  1   -1645852809 0.0 6914.0  -6914.0 6914.0  
4.4470848E7 -1645858447.15  -1645852809 -1.554726368159204E-4   6914.0  
6914.0  4.4477762E7 0.0
--1645852809false   S7UM6KgdxTofi6rwXBFa2a  NULL12520.0 -8.052864E7 
1645852809  1   -1645852809 0.0 12520.0 -12520.012520.0 
8.052864E7  -1645858447.15  -1645852809 -1.554726368159204E-4   12520.0 
12520.0 8.054116E7  0.0
--1645852809false   eNsh5tYaNULLNULLNULL1645852809  
1   -1645852809 0.0 NULLNULLNULLNULL-1645858447.15  
-1645852809 NULLNULLNULLNULLNULL
--1645852809false   iS4P5128HY44wa  NULL3890.0  -2.502048E7 
1645852809  1   -1645852809 0.0 3890.0  -3890.0 3890.0  
2.502048E7  -1645858447.15  -1645852809 -1.554726368159204E-4   3890.0  
3890.0  2.502437E7  0.0
--1645852809false   kro4Xu41bB7hiFa NULL-3277.0 2.1077664E7 
1645852809  1   -1645852809 0.0 -3277.0 3277.0  -3277.0 
-2.1077664E7-1645858447.15  -1645852809 -1.554726368159204E-4   -3277.0 
-3277.0 -2.1080941E70.0
--1645852809false   lJ63qx87BLmdMfa NULL11619.0 -7.4733408E7
1645852809  1   -1645852809 0.0 11619.0 -11619.011619.0 
7.4733408E7 -1645858447.15  -1645852809 -1.554726368159204E-4   11619.0 
11619.0 7.4745027E7 0.0
--1645852809true4gBPJa  NULL13167.0 -8.4690144E71645852809  
1   -1645852809 0.0 13167.0 -13167.013167.0 8.4690144E7 
-1645858447.15  -1645852809 -1.554726368159204E-4   13167.0 13167.0 
8.4703311E7 0.0
--1645852809trueL057p1HPpJsmA3a NULL-9542.0 6.1374144E7 
1645852809  1   -1645852809 0.0 -9542.0 9542.0  -9542.0 
-6.1374144E7-1645858447.15  -1645852809 -1.554726368159204E-4   -9542.0 
-9542.0 -6.1383686E70.0
--1645852809truePMoJ1NvQoAm5a   NULL539.0   -3466848.0  
1645852809  1   -1645852809 0.0 539.0   -539.0  539.0   
3466848.0   -1645858447.15  -1645852809 -1.554726368159204E-4   539.0   
539.0   3467387.0   0.0
--1645852809trueTt484a  NULL754.0   -4849728.0  1645852809  
1   -1645852809 0.0 754.0   -754.0  754.0   4849728.0   
-1645858447.15  -1645852809 -1.554726368159204E-4   754.0   754.0   
4850482.0   0.0
--1645852809truea   NULL-2944.0 1.8935808E7 1645852809  
1   -1645852809 0.0 -2944.0 2944.0  -2944.0 -1.8935808E7
-1645858447.15  -1645852809 -1.554726368159204E-4   -2944.0 -2944.0 
-1.8938752E70.0
--1645852809truea   NULL-5905.0 3.798096E7  1645852809  
1   -1645852809 0.0 -5905.0 5905.0  -5905.0 -3.798096E7 
-1645858447.15  -1645852809 -1.554726368159204E-4   -5905.0 -5905.0 
-3.7986865E70.0
--1645852809truea   NULL4991.0  -3.2102112E71645852809  
1   -1645852809 0.0 4991.0  -4991.0 4991.0  3.2102112E7 
-1645858447.15  -1645852809 -1.554726368159204E-4   4991.0  4991.0  
3.2107103E7 0.0
--1645852809truebBAKio7bAmQq7vIlsc8H14a NULL1949.0  -1.2535968E7
1645852809  1   -1645852809 0.0 1949.0  -1949.0 1949.0  
1.2535968E7 -1645858447.15  -1645852809 -1.554726368159204E-4   1949.0  
1949.0  1.2537917E7 0.0
--1645852809true

[46/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_all.q.out
--
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index e08f1a8,000..ea60414
mode 100644,00..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2443 -1,0 +1,2147 @@@
 +PREHOOK: query: drop table intermediate
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate(key int) partitioned by (p int) 
stored as orc
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate
 +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) 
stored as orc
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate
 +PREHOOK: query: insert into table intermediate partition(p='455') select 
distinct key from src where key >= 0 order by key desc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=455
 +POSTHOOK: query: insert into table intermediate partition(p='455') select 
distinct key from src where key >= 0 order by key desc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=455
 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='456') select 
distinct key from src where key is not null order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=456
 +POSTHOOK: query: insert into table intermediate partition(p='456') select 
distinct key from src where key is not null order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=456
 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='457') select 
distinct key from src where key >= 100 order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=457
 +POSTHOOK: query: insert into table intermediate partition(p='457') select 
distinct key from src where key >= 100 order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=457
 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION 
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) 
stored as orc tblproperties ("transactional"="true", 
"transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) 
stored as orc tblproperties ("transactional"="true", 
"transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: explain insert into table part_mm partition(key_mm=455) 
select key from intermediate
 +PREHOOK: type: QUERY
 +POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) 
select key from intermediate
 +POSTHOOK: type: QUERY
 +STAGE DEPENDENCIES:
 +  Stage-1 is a root stage
 +  Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
 +  Stage-4
 +  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
 +  Stage-2 depends on stages: Stage-0
 +  Stage-3
 +  Stage-5
 +  Stage-6 depends on stages: Stage-5
 +
 +STAGE PLANS:
 +  Stage: Stage-1
 +Map Reduce
 +  Map Operator Tree:
 +  TableScan
 +alias: intermediate
 +Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +Select Operator
 +  expressions: key (type: int)
 +  outputColumnNames: _col0
 +  Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +  File Output Operator
 +compressed: false
 +Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE 
Column stats: NONE
 +table:
 +input format: 
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +output format: 
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +name: default.part_mm
 +

[05/50] [abbrv] hive git commit: HIVE-17392: SharedWorkOptimizer might merge TS operators filtered by not equivalent semijoin operators (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[weiz]

HIVE-17392: SharedWorkOptimizer might merge TS operators filtered by not 
equivalent semijoin operators (Jesus Camacho Rodriguez, reviewed by Ashutosh 
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/262d8f99
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/262d8f99
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/262d8f99

Branch: refs/heads/hive-14535
Commit: 262d8f992e63ac4aa65e36665fb22748546c511c
Parents: 7567119
Author: Jesus Camacho Rodriguez 
Authored: Fri Aug 25 13:25:24 2017 -0700
Committer: Jesus Camacho Rodriguez 
Committed: Fri Aug 25 17:46:15 2017 -0700

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/optimizer/SharedWorkOptimizer.java  |  14 +-
 .../dynamic_semijoin_reduction_sw.q |  60 +++
 .../llap/dynamic_semijoin_reduction_sw.q.out| 518 +++
 4 files changed, 585 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 772113a..37a3757 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -501,6 +501,7 @@ minillaplocal.query.files=acid_globallimit.q,\
   dynamic_semijoin_reduction.q,\
   dynamic_semijoin_reduction_2.q,\
   dynamic_semijoin_reduction_3.q,\
+  dynamic_semijoin_reduction_sw.q,\
   dynpart_sort_opt_vectorization.q,\
   dynpart_sort_optimization.q,\
   dynpart_sort_optimization_acid.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index 01d4f57..37fdb00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -443,8 +443,8 @@ public class SharedWorkOptimizer extends Transform {
 Set ascendants =
 findAscendantWorkOperators(pctx, optimizerCache, op);
 if (ascendants.contains(tsOp2)) {
-  dppsOp1.remove(i);
-  i--;
+  // This should not happen, we cannot merge
+  return false;
 }
   }
 }
@@ -454,8 +454,8 @@ public class SharedWorkOptimizer extends Transform {
 Set ascendants =
 findAscendantWorkOperators(pctx, optimizerCache, op);
 if (ascendants.contains(tsOp1)) {
-  dppsOp2.remove(i);
-  i--;
+  // This should not happen, we cannot merge
+  return false;
 }
   }
 }
@@ -464,9 +464,9 @@ public class SharedWorkOptimizer extends Transform {
   return false;
 }
 // Check if DPP branches are equal
+BitSet bs = new BitSet();
 for (int i = 0; i < dppsOp1.size(); i++) {
   Operator dppOp1 = dppsOp1.get(i);
-  BitSet bs = new BitSet();
   for (int j = 0; j < dppsOp2.size(); j++) {
 if (!bs.get(j)) {
   // If not visited yet
@@ -478,7 +478,7 @@ public class SharedWorkOptimizer extends Transform {
   }
 }
   }
-  if (bs.cardinality() == i) {
+  if (bs.cardinality() < i + 1) {
 return false;
   }
 }
@@ -530,7 +530,6 @@ public class SharedWorkOptimizer extends Transform {
 if (currentOp1.getChildOperators().size() > 1 ||
 currentOp2.getChildOperators().size() > 1) {
   // TODO: Support checking multiple child operators to merge further.
-  discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, 
discardableInputOps));
   discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, 
discardableOps));
   discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, 
retainableOps, discardableInputOps));
   return new SharedResult(retainableOps, discardableOps, 
discardableInputOps, dataSize, maxDataSize);
@@ -539,7 +538,6 @@ public class SharedWorkOptimizer extends Transform {
 currentOp2 = currentOp2.getChildOperators().get(0);
   } else {
 // Bail out
-discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, 
discardableInputOps));
 discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, 
discardableOps));
 discardableInputOps.addAll(gatherDPPBranchOps(pctx, 

[11/50] [abbrv] hive git commit: HIVE-17375: stddev_samp, var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_9.q.out
--
diff --git a/ql/src/test/results/clientpositive/vectorization_9.q.out 
b/ql/src/test/results/clientpositive/vectorization_9.q.out
index 930b476..7af0bbd 100644
--- a/ql/src/test/results/clientpositive/vectorization_9.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_9.q.out
@@ -474,168 +474,168 @@ N6BMOr83ecL NULL1969-12-31 16:00:08.451 NULL
NULL0   NULLNULLNULLNULLNULL0.
 N6Dh6XreCWb0aA4nmDnFOO NULL1969-12-31 16:00:02.351 NULLNULL0   
NULLNULLNULLNULLNULL0.00NULL
 N8222wByj  NULL1969-12-31 16:00:08.451 NULLNULL0   NULL
NULLNULLNULLNULL0.00NULL
 NABd3KhjjaVfcj2Q7SJ46  NULL1969-12-31 16:00:02.351 NULLNULL0   
NULLNULLNULLNULLNULL0.00NULL
-NULL   15601.0 1969-12-31 15:59:43.919 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.07  -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.179 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.394 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.477 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.568 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.571 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.708 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:44.782 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:45.816 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:46.114 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:46.82  -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:46.953 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:47.134 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:47.406 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:47.511 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:47.616 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:47.975 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.052 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.299 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.429 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.552 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.679 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:48.943 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:49.331 -9747614.5639   9747614.56391   
0.0 -0.00.0 15601.0 625.8070356964297   -0.719942   0.0
-NULL   15601.0 1969-12-31 15:59:49.896 

[36/50] [abbrv] hive git commit: HIVE-17415 - Hit error "SemanticException View xxx is corresponding to LIMIT, rather than a SelectOperator." in Hive queries (Deepak Jaiswal, reviewed by Ashutosh Chau

[weiz]

HIVE-17415 - Hit error "SemanticException View xxx is corresponding to LIMIT, 
rather than a SelectOperator." in Hive queries (Deepak Jaiswal, reviewed by 
Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b82d38aa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b82d38aa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b82d38aa

Branch: refs/heads/hive-14535
Commit: b82d38aa38f71a66bee47977c041e7d8a28c8419
Parents: 45f3fac
Author: Deepak Jaiswal 
Authored: Thu Aug 31 21:40:22 2017 -0700
Committer: Vineet Garg 
Committed: Thu Aug 31 21:41:55 2017 -0700

--
 .../test/resources/testconfiguration.properties |  2 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  8 ++-
 .../clientpositive/authorization_view_8.q   | 23 +++
 .../llap/authorization_view_8.q.out | 69 
 4 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index f452341..6504250 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -87,6 +87,7 @@ minillap.shared.query.files=insert_into1.q,\
 minillaplocal.shared.query.files=alter_merge_2_orc.q,\
   alter_merge_orc.q,\
   alter_merge_stats_orc.q,\
+  authorization_view_8.q,\
   auto_join0.q,\
   auto_join1.q,\
   auto_join21.q,\
@@ -462,6 +463,7 @@ minillaplocal.query.files=\
   acid_globallimit.q,\
   acid_vectorization_missing_cols.q,\
   alter_merge_stats_orc.q,\
+  authorization_view_8.q,\
   auto_join30.q,\
   auto_join_filters.q,\
   auto_join_nulls.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e8acabe..6f379da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.LimitOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.exec.RecordReader;
@@ -10547,10 +10548,15 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
 // Recurse over the subqueries to fill the subquery part of the plan
 for (String alias : qb.getSubqAliases()) {
   QBExpr qbexpr = qb.getSubqForAlias(alias);
-  Operator operator = genPlan(qb, qbexpr);
+  Operator operator = genPlan(qb, qbexpr);
   aliasToOpInfo.put(alias, operator);
   if (qb.getViewToTabSchema().containsKey(alias)) {
 // we set viewProjectToTableSchema so that we can leverage 
ColumnPruner.
+if (operator instanceof LimitOperator) {
+  // If create view has LIMIT operator, this can happen
+  // Fetch parent operator
+  operator = operator.getParentOperators().get(0);
+}
 if (operator instanceof SelectOperator) {
   if (this.viewProjectToTableSchema == null) {
 this.viewProjectToTableSchema = new LinkedHashMap<>();

http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/ql/src/test/queries/clientpositive/authorization_view_8.q
--
diff --git a/ql/src/test/queries/clientpositive/authorization_view_8.q 
b/ql/src/test/queries/clientpositive/authorization_view_8.q
new file mode 100644
index 000..82fc97b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/authorization_view_8.q
@@ -0,0 +1,23 @@
+set 
hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
+
+create table my_passwd (
+username string,
+uid int);
+
+insert into my_passwd values
+  ("Deepak", 1),
+  ("Gunther", 2),
+  ("Jason", 3),
+  ("Prasanth", 4),
+  ("Gopal", 5),
+  ("Sergey", 6);
+
+
+set hive.cbo.enable=false;
+create view my_passwd_vw as select * from my_passwd limit 3;
+

[24/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
new file mode 100644
index 000..a003d37
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ReplState {
+  @JsonIgnoreProperties
+  private static final Logger REPL_LOG = LoggerFactory.getLogger("ReplState");
+
+  @JsonIgnoreProperties
+  private static final ObjectMapper mapper = new ObjectMapper(); // 
Thread-safe.
+
+  static {
+mapper.configure(SerializationConfig.Feature.AUTO_DETECT_GETTERS, false);
+mapper.configure(SerializationConfig.Feature.AUTO_DETECT_IS_GETTERS, 
false);
+mapper.configure(SerializationConfig.Feature.AUTO_DETECT_FIELDS, false);
+  }
+
+  public enum LogTag {
+START,
+TABLE_DUMP,
+FUNCTION_DUMP,
+EVENT_DUMP,
+TABLE_LOAD,
+FUNCTION_LOAD,
+EVENT_LOAD,
+END
+  }
+
+  public void log(LogTag tag) {
+try {
+  REPL_LOG.info("REPL::{}: {}", tag.name(), 
mapper.writeValueAsString(this));
+} catch (Exception exception) {
+  REPL_LOG.error("Could not serialize REPL log: {}", 
exception.getMessage());
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
index f40c703..a48a17e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Collections2;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 
 public class Utils {
@@ -64,14 +65,18 @@ public class Utils {
   public static Iterable matchesTbl(Hive db, String dbName, 
String tblPattern)
   throws HiveException {
 if (tblPattern == null) {
-  return Collections2.filter(db.getAllTables(dbName),
-  tableName -> {
-assert tableName != null;
-return !tableName.toLowerCase().startsWith(
-SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
-  });
+  return getAllTables(db, dbName);
 } else {
   return db.getTablesByPattern(dbName, tblPattern);
 }
   }
+
+  public static Collection getAllTables(Hive db, String dbName) throws 
HiveException {
+return Collections2.filter(db.getAllTables(dbName),
+tableName -> {
+  assert tableName != null;
+  return !tableName.toLowerCase().startsWith(
+  
SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
+});
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
new file mode 100644
index 000..1b0ec59
--- /dev/null
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The 

[08/50] [abbrv] hive git commit: HIVE-17205 - add functional support for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)

[weiz]

HIVE-17205 - add functional support for unbucketed tables (Eugene Koifman, 
reviewed by Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6be50b76
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6be50b76
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6be50b76

Branch: refs/heads/hive-14535
Commit: 6be50b76be5956b3c52ed6024fd7b4a3dee65fb6
Parents: 262d8f9
Author: Eugene Koifman 
Authored: Fri Aug 25 20:14:57 2017 -0700
Committer: Eugene Koifman 
Committed: Fri Aug 25 20:15:26 2017 -0700

--
 .../streaming/AbstractRecordWriter.java |   44 +-
 .../hive/hcatalog/streaming/HiveEndPoint.java   |4 +-
 .../apache/hive/hcatalog/streaming/package.html |   21 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |   86 +-
 .../hive/metastore/TestHiveMetaStore.java   |8 +-
 .../apache/hadoop/hive/ql/TestAcidOnTez.java|  367 +++-
 .../hive/ql/TestAcidOnTezWithSplitUpdate.java   |   28 -
 .../test/resources/testconfiguration.properties |4 +-
 .../TransactionalValidationListener.java|   42 +-
 .../org/apache/hadoop/hive/ql/ErrorMsg.java |4 +-
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |   22 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   60 +-
 .../hive/ql/io/orc/OrcRawRecordMerger.java  |  225 +-
 .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java |   14 +-
 .../apache/hadoop/hive/ql/io/orc/OrcSplit.java  |   10 +-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java |   26 +-
 .../optimizer/SortedDynPartitionOptimizer.java  |1 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |6 -
 .../hive/ql/txn/compactor/CompactorMR.java  |3 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  108 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   12 +-
 .../hadoop/hive/ql/TestTxnCommandsBase.java |  162 ++
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |  297 +++
 .../hive/ql/io/orc/TestInputOutputFormat.java   |   28 +-
 .../hive/ql/io/orc/TestOrcRawRecordMerger.java  |   86 +-
 .../queries/clientnegative/create_not_acid.q|2 +-
 .../queries/clientpositive/acid_no_buckets.q|  210 ++
 .../clientnegative/create_not_acid.q.out|4 +-
 .../clientnegative/delete_non_acid_table.q.out  |2 +-
 .../clientnegative/update_non_acid_table.q.out  |2 +-
 .../clientpositive/llap/acid_no_buckets.q.out   | 1976 ++
 31 files changed, 3534 insertions(+), 330 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
index e409e75..4ec10ad 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
@@ -46,6 +46,7 @@ import java.io.IOException;
 
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
@@ -54,19 +55,23 @@ import java.util.Properties;
 public abstract class AbstractRecordWriter implements RecordWriter {
   static final private Logger LOG = 
LoggerFactory.getLogger(AbstractRecordWriter.class.getName());
 
-  final HiveConf conf;
-  final HiveEndPoint endPoint;
+  private final HiveConf conf;
+  private final HiveEndPoint endPoint;
   final Table tbl;
 
-  final IMetaStoreClient msClient;
-  protected final List bucketIds;
-  ArrayList updaters = null;
+  private final IMetaStoreClient msClient;
+  final List bucketIds;
+  private ArrayList updaters = null;
 
-  public final int totalBuckets;
+  private final int totalBuckets;
+  /**
+   * Indicates whether target table is bucketed
+   */
+  private final boolean isBucketed;
 
   private final Path partitionPath;
 
-  final AcidOutputFormat outf;
+  private final AcidOutputFormat outf;
   private Object[] bucketFieldData; // Pre-allocated in constructor. Updated 
on each write.
   private Long curBatchMinTxnId;
   private Long curBatchMaxTxnId;
@@ -109,16 +114,22 @@ public abstract class AbstractRecordWriter implements 
RecordWriter {
 this.tbl = twp.tbl;
 this.partitionPath = twp.partitionPath;
   }
-  this.totalBuckets = tbl.getSd().getNumBuckets();
-  if (totalBuckets <= 0) {
-throw new StreamingException("Cannot stream to table that has not been 
bucketed : "
-  + endPoint);
+  this.isBucketed = 

[37/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/thrift/hive_metastore.thrift
--
diff --cc standalone-metastore/src/main/thrift/hive_metastore.thrift
index 000,7268d53..322e0ec
mode 00,100644..100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@@ -1,0 -1,1612 +1,1613 @@@
+ #!/usr/local/bin/thrift -java
+ 
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ #
+ # Thrift Service that the MetaStore is built on
+ #
+ 
+ include "share/fb303/if/fb303.thrift"
+ 
+ namespace java org.apache.hadoop.hive.metastore.api
+ namespace php metastore
+ namespace cpp Apache.Hadoop.Hive
+ 
+ const string DDL_TIME = "transient_lastDdlTime"
+ 
+ struct Version {
+   1: string version,
+   2: string comments
+ }
+ 
+ struct FieldSchema {
+   1: string name, // name of the field
+   2: string type, // type of the field. primitive types defined above, 
specify list, map for lists & maps
+   3: string comment
+ }
+ 
+ struct SQLPrimaryKey {
+   1: string table_db,// table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: i32 key_seq,// sequence number within primary key
+   5: string pk_name, // primary key name
+   6: bool enable_cstr,   // Enable/Disable
+   7: bool validate_cstr,  // Validate/No validate
+   8: bool rely_cstr  // Rely/No Rely
+ }
+ 
+ struct SQLForeignKey {
+   1: string pktable_db,// primary key table schema
+   2: string pktable_name,  // primary key table name
+   3: string pkcolumn_name, // primary key column name
+   4: string fktable_db,// foreign key table schema
+   5: string fktable_name,  // foreign key table name
+   6: string fkcolumn_name, // foreign key column name
+   7: i32 key_seq,  // sequence within foreign key
+   8: i32 update_rule,  // what happens to foreign key when parent key is 
updated
+   9: i32 delete_rule,  // what happens to foreign key when parent key is 
deleted
+   10: string fk_name,  // foreign key name
+   11: string pk_name,  // primary key name
+   12: bool enable_cstr,// Enable/Disable
+   13: bool validate_cstr,  // Validate/No validate
+   14: bool rely_cstr   // Rely/No Rely
+ }
+ 
+ struct SQLUniqueConstraint {
+   1: string table_db,// table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: i32 key_seq,// sequence number within unique constraint
+   5: string uk_name, // unique key name
+   6: bool enable_cstr,   // Enable/Disable
+   7: bool validate_cstr, // Validate/No validate
+   8: bool rely_cstr  // Rely/No Rely
+ }
+ 
+ struct SQLNotNullConstraint {
+   1: string table_db,// table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: string nn_name, // not null name
+   5: bool enable_cstr,   // Enable/Disable
+   6: bool validate_cstr, // Validate/No validate
+   7: bool rely_cstr  // Rely/No Rely
+ }
+ 
+ struct Type {
+   1: string  name, // one of the types in PrimitiveTypes 
or CollectionTypes or User defined types
+   2: optional string type1,// object type if the name is 'list' 
(LIST_TYPE), key type if the name is 'map' (MAP_TYPE)
+   3: optional string type2,// val type if the name is 'map' 
(MAP_TYPE)
+   4: optional list fields // if the name is one of the user 
defined types
+ }
+ 
+ enum HiveObjectType {
+   GLOBAL = 1,
+   DATABASE = 2,
+   TABLE = 3,
+   PARTITION = 4,
+   COLUMN = 5,
+ }
+ 
+ enum PrincipalType {
+   USER = 1,
+   ROLE = 2,
+   GROUP = 3,
+ }
+ 
+ const string HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__"
+ const string HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__"
+ const string HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__"
+ 
+ enum PartitionEventType {
+   LOAD_DONE = 1,
+ }
+ 
+ // Enums for transaction and lock management 
+ enum TxnState {
+ COMMITTED = 1,
+ ABORTED = 2,
+ OPEN = 3,
+ }
+ 
+ enum 

[06/50] [abbrv] hive git commit: HIVE-17205 - add functional support for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out 
b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
new file mode 100644
index 000..34dd487
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
@@ -0,0 +1,1976 @@
+PREHOOK: query: drop table if exists srcpart_acid
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists srcpart_acid
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) 
PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES 
('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_acid
+POSTHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) 
PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES 
('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_acid
+PREHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from 
srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid
+POSTHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from 
srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE 
[(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE 
[(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE 
[(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE 
[(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE 
[(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE 
[(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE 
[(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE 
[(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as 
integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+ A masked pattern was here 
+POSTHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as 
integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+ A masked pattern was here 
+2008-04-08 11  43  val_43
+2008-04-08 11  413 val_413
+2008-04-08 11  413 val_413
+2008-04-09 11  43  val_43
+2008-04-09 11  413 val_413
+2008-04-09 11  413 val_413
+PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute 
statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Output: 

[18/50] [abbrv] hive git commit: HIVE-17307 Change the metastore to not use the metrics code in hive/common. This closes #235. (Alan Gates, reviewed by Vihang Karajgaonkar)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
--
diff --git 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
new file mode 100644
index 000..259a4db
--- /dev/null
+++ 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+public class TestMetrics {
+
+  @Test
+  public void jsonReporter() throws Exception {
+String jsonFile = System.getProperty("java.io.tmpdir") + 
System.getProperty("file.separator") +
+"TestMetricsOutput.json";
+Configuration conf = MetastoreConf.newMetastoreConf();
+MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_REPORTERS, 
"json");
+MetastoreConf.setVar(conf, 
MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+MetastoreConf.setTimeVar(conf, 
MetastoreConf.ConfVars.METRICS_JSON_FILE_INTERVAL, 1,
+TimeUnit.SECONDS);
+
+Metrics.initialize(conf);
+
+final List words = Arrays.asList("mary", "had", "a", "little", 
"lamb");
+MetricRegistry registry = Metrics.getRegistry();
+registry.register("my-gauge", new Gauge() {
+
+  @Override
+  public Integer getValue() {
+return words.size();
+  }
+});
+
+Counter counter = Metrics.getOrCreateCounter("my-counter");
+counter.inc();
+counter.inc();
+
+Meter meter = registry.meter("my-meter");
+meter.mark();
+Thread.sleep(10);
+meter.mark();
+
+Timer timer = Metrics.getOrCreateTimer("my-timer");
+timer.time(new Callable() {
+  @Override
+  public Long call() throws Exception {
+Thread.sleep(100);
+return 1L;
+  }
+});
+
+// Make sure it has a chance to dump it.
+Thread.sleep(2000);
+
+FileSystem fs = FileSystem.get(conf);
+Path path = new Path(jsonFile);
+Assert.assertTrue(fs.exists(path));
+
+String json = new String(MetricsTestUtils.getFileData(jsonFile, 200, 10));
+MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, 
"my-counter", 2);
+MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, 
"my-meter", 2);
+MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, 
"my-timer", 1);
+MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, 
"my-gauge", 5);
+  }
+
+  @Test
+  public void allReporters() throws Exception {
+String jsonFile = System.getProperty("java.io.tmpdir") + 
System.getProperty("file.separator") +
+"TestMetricsOutput.json";
+Configuration conf = MetastoreConf.newMetastoreConf();
+MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_REPORTERS, 
"json,jmx,console,hadoop");
+MetastoreConf.setVar(conf, 
MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+
+Metrics.initialize(conf);
+
+Assert.assertEquals(4, Metrics.getReporters().size());
+  }
+
+  @Test
+  public void allReportersHiveConfig() throws Exception {
+String jsonFile = System.getProperty("java.io.tmpdir") + 
System.getProperty("file.separator") +
+

[02/50] [abbrv] hive git commit: HIVE-17330 : refactor TezSessionPoolManager to separate its multiple functions (Sergey Shelukhin, reviewed by Gunther Hagleitner)

[weiz]

HIVE-17330 : refactor TezSessionPoolManager to separate its multiple functions 
(Sergey Shelukhin, reviewed by Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/733bc5f0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/733bc5f0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/733bc5f0

Branch: refs/heads/hive-14535
Commit: 733bc5f0280f0f8145b953d67bb50e34af713e04
Parents: 6b10382
Author: sergey 
Authored: Fri Aug 25 11:52:09 2017 -0700
Committer: sergey 
Committed: Fri Aug 25 11:53:05 2017 -0700

--
 .../ql/exec/tez/RestrictedConfigChecker.java|  79 +++
 .../ql/exec/tez/SessionExpirationTracker.java   | 235 +
 .../hadoop/hive/ql/exec/tez/TezSessionPool.java | 165 ++
 .../hive/ql/exec/tez/TezSessionPoolManager.java | 516 +++
 .../hive/ql/exec/tez/TezSessionPoolSession.java | 158 ++
 .../hive/ql/exec/tez/SampleTezSessionState.java |   3 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java|   2 +-
 7 files changed, 704 insertions(+), 454 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
new file mode 100644
index 000..f6b1c1d
--- /dev/null
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class RestrictedConfigChecker {
+  private final static Logger LOG = 
LoggerFactory.getLogger(RestrictedConfigChecker.class);
+  private final List restrictedHiveConf = new ArrayList<>();
+  private final List restrictedNonHiveConf = new ArrayList<>();
+  private final HiveConf initConf;
+
+  RestrictedConfigChecker(HiveConf initConf) {
+this.initConf = initConf;
+String[] restrictedConfigs = HiveConf.getTrimmedStringsVar(initConf,
+ConfVars.HIVE_SERVER2_TEZ_SESSION_RESTRICTED_CONFIGS);
+if (restrictedConfigs == null || restrictedConfigs.length == 0) return;
+HashMap confVars = HiveConf.getOrCreateReverseMap();
+for (String confName : restrictedConfigs) {
+  if (confName == null || confName.isEmpty()) continue;
+  confName = confName.toLowerCase();
+  ConfVars cv = confVars.get(confName);
+  if (cv != null) {
+restrictedHiveConf.add(cv);
+  } else {
+LOG.warn("A restricted config " + confName + " is not recognized as a 
Hive setting.");
+restrictedNonHiveConf.add(confName);
+  }
+}
+  }
+
+  public void validate(HiveConf conf) throws HiveException {
+for (ConfVars var : restrictedHiveConf) {
+  String userValue = HiveConf.getVarWithoutType(conf, var),
+  serverValue = HiveConf.getVarWithoutType(initConf, var);
+  // Note: with some trickery, we could add logic for each type in 
ConfVars; for now the
+  // potential spurious mismatches (e.g. 0 and 0.0 for float) should be 
easy to work around.
+  validateRestrictedConfigValues(var.varname, userValue, serverValue);
+}
+for (String var : restrictedNonHiveConf) {
+  String userValue = conf.get(var), serverValue = initConf.get(var);
+  validateRestrictedConfigValues(var, userValue, serverValue);
+}
+  }
+
+  private void validateRestrictedConfigValues(
+  String var, String userValue, String serverValue) throws HiveException {
+if ((userValue == null) != (serverValue == null)
+ 

[45/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_loaddata.q.out
--
diff --cc ql/src/test/results/clientpositive/mm_loaddata.q.out
index 000,000..b849a88
new file mode 100644
--- /dev/null
+++ b/ql/src/test/results/clientpositive/mm_loaddata.q.out
@@@ -1,0 -1,0 +1,296 @@@
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load0_mm (key string, value string) stored as 
textfile tblproperties("transactional"="true", 
"transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: create table load0_mm (key string, value string) stored as 
textfile tblproperties("transactional"="true", 
"transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table 
load0_mm
++PREHOOK: type: LOAD
++ A masked pattern was here 
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table 
load0_mm
++POSTHOOK: type: LOAD
++ A masked pattern was here 
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++ A masked pattern was here 
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++ A masked pattern was here 
++500
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table 
load0_mm
++PREHOOK: type: LOAD
++ A masked pattern was here 
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table 
load0_mm
++POSTHOOK: type: LOAD
++ A masked pattern was here 
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++ A masked pattern was here 
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++ A masked pattern was here 
++1000
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite 
into table load0_mm
++PREHOOK: type: LOAD
++ A masked pattern was here 
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite 
into table load0_mm
++POSTHOOK: type: LOAD
++ A masked pattern was here 
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++ A masked pattern was here 
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++ A masked pattern was here 
++500
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load0_mm
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load0_mm
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: drop table intermediate2
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table intermediate2
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table intermediate2 (key string, value string) stored 
as textfile
++ A masked pattern was here 
++PREHOOK: type: CREATETABLE
++ A masked pattern was here 
++PREHOOK: Output: database:default
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: create table intermediate2 (key string, value string) stored 
as textfile
++ A masked pattern was here 
++POSTHOOK: type: CREATETABLE
++ A masked pattern was here 
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table 
intermediate2
++PREHOOK: type: LOAD
++ A masked pattern was here 
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table 
intermediate2
++POSTHOOK: type: LOAD
++ A masked pattern was here 
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table 
intermediate2
++PREHOOK: type: LOAD
++ A masked pattern was here 
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table 
intermediate2
++POSTHOOK: type: LOAD
++ A masked pattern was here 
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table 
intermediate2
++PREHOOK: type: LOAD
++ A masked pattern 

[19/50] [abbrv] hive git commit: HIVE-17307 Change the metastore to not use the metrics code in hive/common. This closes #235. (Alan Gates, reviewed by Vihang Karajgaonkar)

[weiz]

HIVE-17307 Change the metastore to not use the metrics code in hive/common.  
This closes #235.  (Alan Gates, reviewed by Vihang Karajgaonkar)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6b9cd52
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6b9cd52
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6b9cd52

Branch: refs/heads/hive-14535
Commit: a6b9cd527b095fc77e4adc839206455bd937dd07
Parents: 77c28af
Author: Alan Gates 
Authored: Wed Aug 30 12:55:54 2017 -0700
Committer: Alan Gates 
Committed: Wed Aug 30 12:55:54 2017 -0700

--
 .../common/metrics/common/MetricsConstant.java  |  14 --
 .../hive/metastore/TestMetaStoreMetrics.java|  78 +++---
 .../hive/metastore/HMSMetricsListener.java  |  47 ++--
 .../hadoop/hive/metastore/HiveMetaStore.java| 124 +-
 .../hadoop/hive/metastore/ObjectStore.java  |  24 +-
 .../hive/metastore/RetryingHMSHandler.java  |   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  22 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |  25 +-
 standalone-metastore/pom.xml|  20 ++
 .../hadoop/hive/metastore/ThreadPool.java   |  63 +
 .../hive/metastore/metrics/JsonReporter.java| 166 +
 .../hive/metastore/metrics/JvmPauseMonitor.java | 222 +
 .../hadoop/hive/metastore/metrics/Metrics.java  | 243 +++
 .../metastore/metrics/MetricsConstants.java |  46 
 .../hive/metastore/metrics/PerfLogger.java  | 194 +++
 .../hive/metastore/metrics/TestMetrics.java | 200 +++
 16 files changed, 1312 insertions(+), 180 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
--
diff --git 
a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
 
b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index b2e78c3..5d28e2d 100644
--- 
a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ 
b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -47,20 +47,6 @@ public class MetricsConstant {
   public static final String SQL_OPERATION_PREFIX = "hs2_sql_operation_";
   public static final String COMPLETED_SQL_OPERATION_PREFIX = 
"hs2_completed_sql_operation_";
 
-  public static final String INIT_TOTAL_DATABASES = "init_total_count_dbs";
-  public static final String INIT_TOTAL_TABLES = "init_total_count_tables";
-  public static final String INIT_TOTAL_PARTITIONS = 
"init_total_count_partitions";
-
-  public static final String CREATE_TOTAL_DATABASES = "create_total_count_dbs";
-  public static final String CREATE_TOTAL_TABLES = "create_total_count_tables";
-  public static final String CREATE_TOTAL_PARTITIONS = 
"create_total_count_partitions";
-
-  public static final String DELETE_TOTAL_DATABASES = "delete_total_count_dbs";
-  public static final String DELETE_TOTAL_TABLES = "delete_total_count_tables";
-  public static final String DELETE_TOTAL_PARTITIONS = 
"delete_total_count_partitions";
-
-  public static final String DIRECTSQL_ERRORS = "directsql_errors";
-
   // The number of Hive operations that are waiting to enter the compile block
   public static final String WAITING_COMPILE_OPS = "waiting_compile_ops";
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index 9b6cab3..e1cb1f3 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -18,14 +18,13 @@
 package org.apache.hadoop.hive.metastore;
 
 import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
 import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
 import 

[44/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
--
diff --cc standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 000,5c09fdd..3e4c754
mode 00,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@@ -1,0 -1,21817 +1,21819 @@@
+ /**
+  * Autogenerated by Thrift Compiler (0.9.3)
+  *
+  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+  *  @generated
+  */
+ #include "hive_metastore_types.h"
+ 
+ #include 
+ #include 
+ 
+ #include 
+ 
+ namespace Apache { namespace Hadoop { namespace Hive {
+ 
+ int _kHiveObjectTypeValues[] = {
+   HiveObjectType::GLOBAL,
+   HiveObjectType::DATABASE,
+   HiveObjectType::TABLE,
+   HiveObjectType::PARTITION,
+   HiveObjectType::COLUMN
+ };
+ const char* _kHiveObjectTypeNames[] = {
+   "GLOBAL",
+   "DATABASE",
+   "TABLE",
+   "PARTITION",
+   "COLUMN"
+ };
+ const std::map 
_HiveObjectType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, 
_kHiveObjectTypeValues, _kHiveObjectTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kPrincipalTypeValues[] = {
+   PrincipalType::USER,
+   PrincipalType::ROLE,
+   PrincipalType::GROUP
+ };
+ const char* _kPrincipalTypeNames[] = {
+   "USER",
+   "ROLE",
+   "GROUP"
+ };
+ const std::map 
_PrincipalType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kPrincipalTypeValues, _kPrincipalTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kPartitionEventTypeValues[] = {
+   PartitionEventType::LOAD_DONE
+ };
+ const char* _kPartitionEventTypeNames[] = {
+   "LOAD_DONE"
+ };
+ const std::map 
_PartitionEventType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, 
_kPartitionEventTypeValues, _kPartitionEventTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kTxnStateValues[] = {
+   TxnState::COMMITTED,
+   TxnState::ABORTED,
+   TxnState::OPEN
+ };
+ const char* _kTxnStateNames[] = {
+   "COMMITTED",
+   "ABORTED",
+   "OPEN"
+ };
+ const std::map 
_TxnState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kTxnStateValues, 
_kTxnStateNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kLockLevelValues[] = {
+   LockLevel::DB,
+   LockLevel::TABLE,
+   LockLevel::PARTITION
+ };
+ const char* _kLockLevelNames[] = {
+   "DB",
+   "TABLE",
+   "PARTITION"
+ };
+ const std::map 
_LockLevel_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kLockLevelValues, _kLockLevelNames), ::apache::thrift::TEnumIterator(-1, NULL, 
NULL));
+ 
+ int _kLockStateValues[] = {
+   LockState::ACQUIRED,
+   LockState::WAITING,
+   LockState::ABORT,
+   LockState::NOT_ACQUIRED
+ };
+ const char* _kLockStateNames[] = {
+   "ACQUIRED",
+   "WAITING",
+   "ABORT",
+   "NOT_ACQUIRED"
+ };
+ const std::map 
_LockState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, 
_kLockStateValues, _kLockStateNames), ::apache::thrift::TEnumIterator(-1, NULL, 
NULL));
+ 
+ int _kLockTypeValues[] = {
+   LockType::SHARED_READ,
+   LockType::SHARED_WRITE,
+   LockType::EXCLUSIVE
+ };
+ const char* _kLockTypeNames[] = {
+   "SHARED_READ",
+   "SHARED_WRITE",
+   "EXCLUSIVE"
+ };
+ const std::map 
_LockType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kLockTypeValues, 
_kLockTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kCompactionTypeValues[] = {
+   CompactionType::MINOR,
+   CompactionType::MAJOR
+ };
+ const char* _kCompactionTypeNames[] = {
+   "MINOR",
+   "MAJOR"
+ };
+ const std::map 
_CompactionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, 
_kCompactionTypeValues, _kCompactionTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kGrantRevokeTypeValues[] = {
+   GrantRevokeType::GRANT,
+   GrantRevokeType::REVOKE
+ };
+ const char* _kGrantRevokeTypeNames[] = {
+   "GRANT",
+   "REVOKE"
+ };
+ const std::map 
_GrantRevokeType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, 
_kGrantRevokeTypeValues, _kGrantRevokeTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kDataOperationTypeValues[] = {
+   DataOperationType::SELECT,
+   DataOperationType::INSERT,
+   DataOperationType::UPDATE,
+   DataOperationType::DELETE,
+   DataOperationType::UNSET,
+   DataOperationType::NO_TXN
+ };
+ const char* _kDataOperationTypeNames[] = {
+   "SELECT",
+   "INSERT",
+   "UPDATE",
+   "DELETE",
+   "UNSET",
+   "NO_TXN"
+ };
+ const std::map 
_DataOperationType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(6, 
_kDataOperationTypeValues, _kDataOperationTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int 

[30/50] [abbrv] hive git commit: HIVE-17006 : LLAP: Parquet caching v1 (Sergey Shelukhin, reviewed by Gunther Hagleitner)

[weiz]

http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_types.q
--
diff --git a/ql/src/test/queries/clientpositive/parquet_types.q 
b/ql/src/test/queries/clientpositive/parquet_types.q
index db37d2e..1a8844b 100644
--- a/ql/src/test/queries/clientpositive/parquet_types.q
+++ b/ql/src/test/queries/clientpositive/parquet_types.q
@@ -1,4 +1,5 @@
 set hive.mapred.mode=nonstrict;
+set hive.llap.cache.allow.synthetic.fileid=true;
 DROP TABLE parquet_types_staging;
 DROP TABLE parquet_types;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
--
diff --git a/ql/src/test/queries/clientpositive/parquet_types_vectorization.q 
b/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
index bb0e5b2..1f353ae 100644
--- a/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
+++ b/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
@@ -7,6 +7,7 @@ set hive.vectorized.execution.reduce.enabled=true;
 set hive.vectorized.use.row.serde.deserialize=true;
 set hive.vectorized.use.vector.serde.deserialize=true;
 set hive.vectorized.execution.reduce.groupby.enabled = true;
+set hive.llap.cache.allow.synthetic.fileid=true;
 
 CREATE TABLE parquet_types_staging (
   cint int,

http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/vectorized_parquet.q
--
diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet.q 
b/ql/src/test/queries/clientpositive/vectorized_parquet.q
index db02ec0..1efc60e 100644
--- a/ql/src/test/queries/clientpositive/vectorized_parquet.q
+++ b/ql/src/test/queries/clientpositive/vectorized_parquet.q
@@ -1,6 +1,7 @@
 set hive.explain.user=false;
 set hive.exec.submitviachild=false;
 set hive.exec.submit.local.task.via.child=false;
+set hive.llap.cache.allow.synthetic.fileid=true;
 
 create table if not exists alltypes_parquet (
   cint int, 

http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
--
diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q 
b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
index 7467cb3..63f811b 100644
--- a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
+++ b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
@@ -1,5 +1,6 @@
 set hive.mapred.mode=nonstrict;
 set hive.explain.user=false;
+set hive.llap.cache.allow.synthetic.fileid=true;
 
 DROP TABLE parquet_types_staging;
 DROP TABLE parquet_types;

http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out 
b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
index aecbcfd..01458fe 100644
--- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
@@ -140,7 +140,7 @@ STAGE PLANS:
 Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint)
 Execution mode: llap
-LLAP IO: no inputs
+LLAP IO: all inputs (cache only)
 Reducer 2 
 Execution mode: llap
 Reduce Operator Tree:
@@ -198,7 +198,7 @@ STAGE PLANS:
 Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
 value expressions: _col0 (type: bigint)
 Execution mode: llap
-LLAP IO: no inputs
+LLAP IO: all inputs (cache only)
 Reducer 2 
 Execution mode: llap
 Reduce Operator Tree:
@@ -540,7 +540,7 @@ STAGE PLANS:
 sort order: ++
 Statistics: Num rows: 29 Data size: 319 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
-LLAP IO: no inputs
+LLAP IO: all inputs (cache only)
 Reducer 2 
 Execution mode: llap
 Reduce Operator Tree:
@@ -606,7 +606,7 @@ STAGE PLANS:
 sort order: ++
 Statistics: Num rows: 29 Data size: 319 Basic stats: 
COMPLETE Column stats: NONE
 Execution mode: llap
-LLAP IO: no inputs
+LLAP IO: all inputs (cache only)
 Reducer 2 
 Execution mode: llap
 Reduce Operator Tree:
@@ -739,7 

[35/50] [abbrv] hive git commit: HIVE-17385: incremental repl error for non-native tables

[weiz]

HIVE-17385: incremental repl error for non-native tables


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45f3fac5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45f3fac5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45f3fac5

Branch: refs/heads/hive-14535
Commit: 45f3fac554ada3cd4b15a7bf3c9f0047c65fa616
Parents: 601c764
Author: Daniel Dai 
Authored: Thu Aug 31 21:08:02 2017 -0700
Committer: Daniel Dai 
Committed: Thu Aug 31 21:08:02 2017 -0700

--
 .../apache/hadoop/hive/ql/parse/EximUtil.java   | 30 
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |  5 
 .../hive/ql/parse/repl/dump/TableExport.java|  7 +
 .../repl/dump/events/CreateTableHandler.java|  8 --
 .../ql/parse/repl/dump/io/TableSerializer.java  |  7 ++---
 5 files changed, 44 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index 22094c0..40c34bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer;
 import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
 import org.apache.hadoop.hive.ql.parse.repl.dump.io.ReplicationSpecSerializer;
@@ -385,4 +386,33 @@ public class EximUtil {
 };
   }
 
+  /**
+   * Verify if a table should be exported or not
+   */
+  public static Boolean shouldExportTable(ReplicationSpec replicationSpec, 
Table tableHandle) throws SemanticException {
+if (replicationSpec == null)
+{
+  replicationSpec = new ReplicationSpec();
+}
+
+if (replicationSpec.isNoop())
+{
+  return false;
+}
+
+if (tableHandle == null)
+{
+  return false;
+}
+
+if (replicationSpec.isInReplicationScope()) {
+  return !(tableHandle == null || tableHandle.isTemporary() || 
tableHandle.isNonNative());
+}
+
+if (tableHandle.isNonNative()) {
+  throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
+}
+
+return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index f8d5c8d..7f3460f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -197,6 +197,11 @@ public class ImportSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
 }
 
+if (rv.getTable() == null) {
+  // nothing to do here, silently return.
+  return false;
+}
+
 ReplicationSpec replicationSpec = rv.getReplicationSpec();
 if (replicationSpec.isNoop()){
   // nothing to do here, silently return.

http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 7533a39..e490633 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -156,12 +156,7 @@ public class TableExport {
   }
 
   private boolean shouldExport() throws SemanticException {
-if (replicationSpec.isInReplicationScope()) {
-  return !(tableSpec.tableHandle.isTemporary() || 
tableSpec.tableHandle.isNonNative());
-} else if (tableSpec.tableHandle.isNonNative()) {
-  throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
-}
-return true;
+return EximUtil.shouldExportTable(replicationSpec, tableSpec.tableHandle);
   }
 
   /**

[29/50] [abbrv] hive git commit: HIVE-17216: Additional qtests for HoS DPP (Sahil Takiar, reviewed by Sergio Pena)

[weiz]

HIVE-17216: Additional qtests for HoS DPP (Sahil Takiar, reviewed by Sergio 
Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/642acdf7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/642acdf7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/642acdf7

Branch: refs/heads/hive-14535
Commit: 642acdf76a46d7bb5bd036510d5bcdaa85e9f613
Parents: 3573549
Author: Sahil Takiar 
Authored: Thu Aug 31 09:05:16 2017 -0700
Committer: Sahil Takiar 
Committed: Thu Aug 31 09:06:10 2017 -0700

--
 .../test/resources/testconfiguration.properties |1 +
 .../clientpositive/spark_constprog_dpp.q|   17 +
 .../spark_dynamic_partition_pruning.q   |   40 +-
 .../spark_dynamic_partition_pruning_3.q |  236 ++-
 .../spark/spark_constprog_dpp.q.out |  138 ++
 .../spark/spark_dynamic_partition_pruning.q.out |  351 ++--
 .../spark_dynamic_partition_pruning_3.q.out | 1781 --
 7 files changed, 2272 insertions(+), 292 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index fa6a2aa..6f2efa7 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1403,6 +1403,7 @@ 
miniSparkOnYarn.only.query.files=spark_combine_equivalent_work.q,\
   spark_dynamic_partition_pruning_2.q,\
   spark_dynamic_partition_pruning_3.q,\
   spark_dynamic_partition_pruning_mapjoin_only.q,\
+  spark_constprog_dpp.q,\
   dynamic_rdd_cache.q, \
   spark_multi_insert_parallel_orderby.q,\
   spark_explainuser_1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
--
diff --git a/ql/src/test/queries/clientpositive/spark_constprog_dpp.q 
b/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
new file mode 100644
index 000..cbe6aa2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
@@ -0,0 +1,17 @@
+set hive.mapred.mode=nonstrict;
+set hive.optimize.constant.propagation=true;
+set hive.spark.dynamic.partition.pruning=true;
+
+drop table if exists tb1;
+create table tb1 (id int);
+
+drop table if exists tb2;
+create table tb2 (id smallint);
+
+explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id;

http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
--
diff --git 
a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q 
b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
index 78e7515..71a7399 100644
--- a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
+++ b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
@@ -16,7 +16,7 @@ create table srcpart_hour as select hr as hr, hr as hour from 
srcpart group by h
 create table srcpart_date_hour as select ds as ds, ds as `date`, hr as hr, hr 
as hour from srcpart group by ds, hr;
 create table srcpart_double_hour as select (hr*2) as hr, hr as hour from 
srcpart group by hr;
 
--- single column, single key
+-- single column, single key -- join a partitioned table to a non-partitioned 
table, static filter on the non-partitioned table
 EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
 select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
 set hive.spark.dynamic.partition.pruning=false;
@@ -41,8 +41,8 @@ select count(*) from srcpart join srcpart_date on 
abs(negative(cast(concat(cast(
 EXPLAIN select count(*) from srcpart join srcpart_date on cast(day(srcpart.ds) 
as smallint) = cast(day(srcpart_date.ds) as decimal) where srcpart_date.`date` 
= '2008-04-08';
 select count(*) from srcpart join srcpart_date on cast(day(srcpart.ds) as 
smallint) = cast(day(srcpart_date.ds) as decimal) where srcpart_date.`date` = 
'2008-04-08';
 
--- multiple sources, single key
-EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = 
srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr) 
+-- multiple sources, single key -- filter partitioned table on both 
partitioned columns via join with non-partitioned table
+EXPLAIN select count(*) from srcpart join 

[10/50] [abbrv] hive git commit: HIVE-17309 alter partition onto a table not in current database throw InvalidOperationException (Wang Haihua via Alan Gates)

[weiz]

HIVE-17309 alter partition onto a table not in current database throw 
InvalidOperationException (Wang Haihua via Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e352ef40
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e352ef40
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e352ef40

Branch: refs/heads/hive-14535
Commit: e352ef409571383bcd546f399a17fc7c8a6e6bf0
Parents: 3fc131c
Author: Alan Gates 
Authored: Mon Aug 28 12:51:31 2017 -0700
Committer: Alan Gates 
Committed: Mon Aug 28 12:51:31 2017 -0700

--
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  2 +-
 .../alter_partition_onto_nocurrent_db.q | 15 
 .../alter_partition_onto_nocurrent_db.q.out | 78 
 3 files changed, 94 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 364db27..acc2390 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3646,7 +3646,7 @@ public class DDLTask extends Task implements 
Serializable {
   if (allPartitions == null) {
 db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), 
alterTbl.getEnvironmentContext());
   } else {
-db.alterPartitions(alterTbl.getOldName(), allPartitions, 
alterTbl.getEnvironmentContext());
+db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), 
allPartitions, alterTbl.getEnvironmentContext());
   }
   // Add constraints if necessary
   addConstraints(db, alterTbl);

http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
--
diff --git 
a/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q 
b/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
new file mode 100644
index 000..017fd44
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
@@ -0,0 +1,15 @@
+CREATE DATABASE test_db_nocurr;
+
+CREATE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) 
PARTITIONED BY (ds STRING) STORED AS SEQUENCEFILE;
+
+INSERT OVERWRITE TABLE 
test_db_nocurr.test_table_for_alter_partition_nocurrentdb 
PARTITION(ds='eleme_haihua') SELECT 1;
+
+desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb 
partition(ds='eleme_haihua');
+
+ALTER TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb 
partition(ds='eleme_haihua') CHANGE COLUMN a a_new BOOLEAN;
+
+desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb 
partition(ds='eleme_haihua');
+
+DROP TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb;
+
+DROP DATABASE test_db_nocurr;

http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out 
b/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
new file mode 100644
index 000..f109bcc
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
@@ -0,0 +1,78 @@
+PREHOOK: query: CREATE DATABASE test_db_nocurr
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:test_db_nocurr
+POSTHOOK: query: CREATE DATABASE test_db_nocurr
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:test_db_nocurr
+PREHOOK: query: CREATE TABLE 
test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) PARTITIONED 
BY (ds STRING) STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_db_nocurr
+PREHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: query: CREATE TABLE 
test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) PARTITIONED 
BY (ds STRING) STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_db_nocurr
+POSTHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+PREHOOK: query: INSERT OVERWRITE TABLE 
test_db_nocurr.test_table_for_alter_partition_nocurrentdb 
PARTITION(ds='eleme_haihua') SELECT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: 
test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+POSTHOOK: 

[26/50] [abbrv] hive git commit: HIVE-17408 : replication distcp should only be invoked if number of files AND file size cross configured limits (Anishek Agarwal reviewed by Thejas Nair)

[weiz]

HIVE-17408 : replication distcp should only be invoked if number of files AND 
file size cross configured limits (Anishek Agarwal reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2e5ead14
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2e5ead14
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2e5ead14

Branch: refs/heads/hive-14535
Commit: 2e5ead142f88333899223aaf810976c4dcf02efb
Parents: 70cb7f0
Author: Anishek Agarwal 
Authored: Wed Aug 30 17:43:01 2017 -0700
Committer: Thejas M Nair 
Committed: Wed Aug 30 17:43:01 2017 -0700

--
 .../hadoop/hive/ql/parse/repl/CopyUtils.java|  4 +-
 .../hive/ql/parse/repl/CopyUtilsTest.java   | 47 
 2 files changed, 49 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2e5ead14/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
index db923e3..28e7bcb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
@@ -108,8 +108,8 @@ public class CopyUtils {
 return true;
   }
 
-  private boolean limitReachedForLocalCopy(long size, long numberOfFiles) {
-boolean result = size > maxCopyFileSize || numberOfFiles > 
maxNumberOfFiles;
+  boolean limitReachedForLocalCopy(long size, long numberOfFiles) {
+boolean result = size > maxCopyFileSize && numberOfFiles > 
maxNumberOfFiles;
 if (result) {
   LOG.info("Source is {} bytes. (MAX: {})", size, maxCopyFileSize);
   LOG.info("Source is {} files. (MAX: {})", numberOfFiles, 
maxNumberOfFiles);

http://git-wip-us.apache.org/repos/asf/hive/blob/2e5ead14/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java 
b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
new file mode 100644
index 000..e643d8f
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+
+public class CopyUtilsTest {
+  /*
+  Distcp currently does not copy a single file in a distributed manner hence 
we dont care about
+  the size of file, if there is only file, we dont want to launch distcp.
+   */
+  @Test
+  public void distcpShouldNotBeCalledOnlyForOneFile() {
+HiveConf conf = new HiveConf();
+conf.setLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE, 1);
+CopyUtils copyUtils = new CopyUtils("", conf);
+long MB_128 = 128 * 1024 * 1024;
+assertFalse(copyUtils.limitReachedForLocalCopy(MB_128, 1L));
+  }
+
+  @Test
+  public void distcpShouldNotBeCalledForSmallerFileSize() {
+HiveConf conf = new HiveConf();
+CopyUtils copyUtils = new CopyUtils("", conf);
+long MB_16 = 16 * 1024 * 1024;
+assertFalse(copyUtils.limitReachedForLocalCopy(MB_16, 100L));
+  }
+}
\ No newline at end of file