hive git commit: HIVE-18614 : Fix sys db creation in Hive (Miklos Gergely via Ashutosh Chauhan)

2018-02-08 Thread hashutosh 
Repository: hive
Updated Branches:
  refs/heads/master 3464df457 -> c34938b9d


HIVE-18614 : Fix sys db creation in Hive (Miklos Gergely via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c34938b9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c34938b9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c34938b9

Branch: refs/heads/master
Commit: c34938b9dfef8557b8329cad1340a15b7b757bd2
Parents: 3464df4
Author: Miklos Gergely 
Authored: Thu Feb 8 23:25:24 2018 -0800
Committer: Ashutosh Chauhan 
Committed: Thu Feb 8 23:25:24 2018 -0800

--
 .../org/apache/hive/beeline/HiveSchemaTool.java | 23 
 .../org/apache/hive/beeline/TestSchemaTool.java |  8 ++--
 .../upgrade/hive/hive-schema-3.0.0.hive.sql |  4 +-
 .../hive/metastore/MetaStoreSchemaInfo.java | 11 +++---
 .../hive/metastore/tools/HiveSchemaHelper.java  | 39 
 5 files changed, 42 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c34938b9/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
--
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java 
b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index fc0887b..0dd594e 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -99,7 +99,7 @@ public class HiveSchemaTool {
 this.hiveConf = hiveConf;
 this.dbType = dbType;
 this.metaDbType = metaDbType;
-this.needsQuotedIdentifier = 
getDbCommandParser(dbType).needsQuotedIdentifier();
+this.needsQuotedIdentifier = getDbCommandParser(dbType, 
metaDbType).needsQuotedIdentifier();
 this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, 
hiveHome, dbType);
   }
 
@@ -151,20 +151,13 @@ public class HiveSchemaTool {
 System.exit(1);
   }
 
-  Connection getConnectionToMetastore(boolean printInfo)
-  throws HiveMetaException {
-return HiveSchemaHelper.getConnectionToMetastore(userName,
-passWord, url, driver, printInfo, hiveConf);
+  Connection getConnectionToMetastore(boolean printInfo) throws 
HiveMetaException {
+return HiveSchemaHelper.getConnectionToMetastore(userName, passWord, url, 
driver, printInfo, hiveConf,
+null);
   }
 
   private NestedScriptParser getDbCommandParser(String dbType, String 
metaDbType) {
-return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName,
-   passWord, hiveConf, metaDbType);
-  }
-
-  private NestedScriptParser getDbCommandParser(String dbType) {
-return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName,
-   passWord, hiveConf, null);
+return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName, 
passWord, hiveConf, metaDbType);
   }
 
   /***
@@ -511,7 +504,7 @@ public class HiveSchemaTool {
 
   private MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
 return new MetaStoreConnectionInfo(userName, passWord, url, driver, 
printInfo, hiveConf,
-dbType);
+dbType, metaDbType);
   }
   /**
* Perform metastore schema upgrade
@@ -593,7 +586,7 @@ public class HiveSchemaTool {
 Connection conn = getConnectionToMetastore(false);
 boolean success = true;
 try {
-  if (validateSchemaVersions(conn)) {
+  if (validateSchemaVersions()) {
 System.out.println("[SUCCESS]\n");
   } else {
 success = false;
@@ -701,7 +694,7 @@ public class HiveSchemaTool {
 }
   }
 
-  boolean validateSchemaVersions(Connection conn) throws HiveMetaException {
+  boolean validateSchemaVersions() throws HiveMetaException {
 System.out.println("Validating schema version");
 try {
   String newSchemaVersion = 
metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));

http://git-wip-us.apache.org/repos/asf/hive/blob/c34938b9/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java 
b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
index 25bd71a..100ec91 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
@@ -233,14 +233,14 @@ public class TestSchemaTool extends TestCase {
   */
  public void testValidateSchemaVersions() throws Exception {
schemaTool.doInit();
-   boolean isValid = schemaTool.validateSchemaVersions(conn);
+

[5/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out 
b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
index d4472cf..054b0d0 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
@@ -1,8 +1,8 @@
-PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_small
-POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_small
@@ -23,11 +23,27 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/sm
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS 
TEXTFILE
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_small@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_small@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_small@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_small@ds=2008-04-08
+PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS 
TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_big
@@ -48,22 +64,6 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/bi
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
 PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/00_0' INTO TABLE bucket_big 
partition(ds='2008-04-09')
 PREHOOK: type: LOAD
  A masked pattern was here 
@@ -81,22 +81,6 @@

[7/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
Revert "HIVE-18350 : load data should rename files consistent with insert 
statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

This reverts commit 6e9b63e48b4f34ba26a6eefb354b0c94ee82256c.


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3464df45
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3464df45
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3464df45

Branch: refs/heads/master
Commit: 3464df45763dd63e20ed1e70a872a600d572c9db
Parents: 1e74aca
Author: Deepak Jaiswal 
Authored: Thu Feb 8 13:11:49 2018 -0800
Committer: Deepak Jaiswal 
Committed: Thu Feb 8 13:13:00 2018 -0800

--
 .../hive/ql/exec/tez/CustomPartitionVertex.java |   71 +-
 .../ql/exec/tez/CustomVertexConfiguration.java  |   31 +-
 .../hadoop/hive/ql/exec/tez/DagUtils.java   |   16 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   |   16 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   54 +-
 .../annotation/OpTraitsRulesProcFactory.java|   35 +-
 .../optimizer/spark/SparkMapJoinOptimizer.java  |3 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java |   43 -
 .../apache/hadoop/hive/ql/plan/OpTraits.java|   22 +-
 .../hadoop/hive/ql/metadata/TestHive.java   |9 +-
 .../clientpositive/auto_sortmerge_join_2.q  |   12 +-
 .../clientpositive/auto_sortmerge_join_4.q  |6 +-
 .../clientpositive/auto_sortmerge_join_5.q  |   10 +-
 .../clientpositive/auto_sortmerge_join_7.q  |8 +-
 .../bucket_mapjoin_mismatch1.q.out  |  170 +
 .../clientpositive/auto_sortmerge_join_2.q.out  |  196 +-
 .../clientpositive/auto_sortmerge_join_4.q.out  |  104 +-
 .../clientpositive/auto_sortmerge_join_5.q.out  |  158 +-
 .../clientpositive/auto_sortmerge_join_7.q.out  |  168 +-
 .../llap/auto_sortmerge_join_2.q.out|  132 +-
 .../llap/auto_sortmerge_join_4.q.out|   74 +-
 .../llap/auto_sortmerge_join_5.q.out|  112 +-
 .../llap/auto_sortmerge_join_7.q.out|  114 +-
 .../spark/auto_sortmerge_join_2.q.out   |  104 +-
 .../spark/auto_sortmerge_join_4.q.out   |   74 +-
 .../spark/auto_sortmerge_join_5.q.out   |  118 +-
 .../spark/auto_sortmerge_join_7.q.out   |  114 +-
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  | 2240 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 3882 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   32 +-
 .../hive/metastore/api/BucketingVersion.java|   48 -
 .../apache/hadoop/hive/metastore/api/Table.java |  230 +-
 .../src/gen/thrift/gen-php/metastore/Types.php  |   57 -
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   45 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   17 +-
 .../hadoop/hive/metastore/ObjectStore.java  |5 +-
 .../hadoop/hive/metastore/model/MTable.java |   37 +-
 .../src/main/thrift/hive_metastore.thrift   |   11 +-
 .../hive/metastore/cache/TestCachedStore.java   |   18 +-
 .../TestTablesCreateDropAlterTruncate.java  |   19 +-
 40 files changed, 3942 insertions(+), 4673 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index ef148d9..26afe90 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -108,15 +108,14 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
   private final Multimap bucketToTaskMap = 
HashMultimap. create();
 
   private final Map> 
inputToGroupedSplitMap =
-  new HashMap<>();
+  new HashMap>();
 
   private int numInputsAffectingRootInputSpecUpdate = 1;
   private int numInputsSeenSoFar = 0;
   private final Map emMap = 
Maps.newHashMap();
   private final List finalSplits = Lists.newLinkedList();
   private final Map inputNameInputSpecMap =
-  new HashMap<>();
-  private Map inputToBucketMap;
+  new HashMap();
 
   public CustomPartitionVertex(VertexManagerPluginContext context) {
 super(context);
@@ -138,7 +137,6 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
 this.mainWorkName = vertexConf.getInputName();
 this.vertexType = vertexConf.getVertexType();

[1/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
Repository: hive
Updated Branches:
  refs/heads/master 1e74aca8d -> 3464df457


http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h 
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 44a2c83..4c09bc8 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -132,16 +132,6 @@ struct EventRequestType {
 
 extern const std::map _EventRequestType_VALUES_TO_NAMES;
 
-struct BucketingVersion {
-  enum type {
-INVALID_BUCKETING = 0,
-JAVA_BUCKETING = 1,
-MURMUR_BUCKETING = 2
-  };
-};
-
-extern const std::map _BucketingVersion_VALUES_TO_NAMES;
-
 struct FunctionType {
   enum type {
 JAVA = 1
@@ -2382,7 +2372,7 @@ inline std::ostream& operator<<(std::ostream& out, const 
StorageDescriptor& obj)
 }
 
 typedef struct _Table__isset {
-  _Table__isset() : tableName(false), dbName(false), owner(false), 
createTime(false), lastAccessTime(false), retention(false), sd(false), 
partitionKeys(false), parameters(false), viewOriginalText(false), 
viewExpandedText(false), tableType(false), privileges(false), temporary(true), 
rewriteEnabled(false), creationMetadata(false), bucketingVersion(true), 
loadInBucketedTable(true) {}
+  _Table__isset() : tableName(false), dbName(false), owner(false), 
createTime(false), lastAccessTime(false), retention(false), sd(false), 
partitionKeys(false), parameters(false), viewOriginalText(false), 
viewExpandedText(false), tableType(false), privileges(false), temporary(true), 
rewriteEnabled(false), creationMetadata(false) {}
   bool tableName :1;
   bool dbName :1;
   bool owner :1;
@@ -2399,8 +2389,6 @@ typedef struct _Table__isset {
   bool temporary :1;
   bool rewriteEnabled :1;
   bool creationMetadata :1;
-  bool bucketingVersion :1;
-  bool loadInBucketedTable :1;
 } _Table__isset;
 
 class Table {
@@ -2408,9 +2396,7 @@ class Table {
 
   Table(const Table&);
   Table& operator=(const Table&);
-  Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), 
retention(0), viewOriginalText(), viewExpandedText(), tableType(), 
temporary(false), rewriteEnabled(0), 
bucketingVersion((BucketingVersion::type)1), loadInBucketedTable(false) {
-bucketingVersion = (BucketingVersion::type)1;
-
+  Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), 
retention(0), viewOriginalText(), viewExpandedText(), tableType(), 
temporary(false), rewriteEnabled(0) {
   }
 
   virtual ~Table() throw();
@@ -2430,8 +2416,6 @@ class Table {
   bool temporary;
   bool rewriteEnabled;
   CreationMetadata creationMetadata;
-  BucketingVersion::type bucketingVersion;
-  bool loadInBucketedTable;
 
   _Table__isset __isset;
 
@@ -2467,10 +2451,6 @@ class Table {
 
   void __set_creationMetadata(const CreationMetadata& val);
 
-  void __set_bucketingVersion(const BucketingVersion::type val);
-
-  void __set_loadInBucketedTable(const bool val);
-
   bool operator == (const Table & rhs) const
   {
 if (!(tableName == rhs.tableName))
@@ -2513,14 +2493,6 @@ class Table {
   return false;
 else if (__isset.creationMetadata && !(creationMetadata == 
rhs.creationMetadata))
   return false;
-if (__isset.bucketingVersion != rhs.__isset.bucketingVersion)
-  return false;
-else if (__isset.bucketingVersion && !(bucketingVersion == 
rhs.bucketingVersion))
-  return false;
-if (__isset.loadInBucketedTable != rhs.__isset.loadInBucketedTable)
-  return false;
-else if (__isset.loadInBucketedTable && !(loadInBucketedTable == 
rhs.loadInBucketedTable))
-  return false;
 return true;
   }
   bool operator != (const Table ) const {

http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
 
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
deleted file mode 100644
index b7de161..000
--- 
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-package org.apache.hadoop.hive.metastore.api;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum BucketingVersion implements org.apache.thrift.TEnum {
-  INVALID_BUCKETING(0),
-

[2/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp 
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 2169265..aadf8f1 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -149,18 +149,6 @@ const char* _kEventRequestTypeNames[] = {
 };
 const std::map 
_EventRequestType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kEventRequestTypeValues, _kEventRequestTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
 
-int _kBucketingVersionValues[] = {
-  BucketingVersion::INVALID_BUCKETING,
-  BucketingVersion::JAVA_BUCKETING,
-  BucketingVersion::MURMUR_BUCKETING
-};
-const char* _kBucketingVersionNames[] = {
-  "INVALID_BUCKETING",
-  "JAVA_BUCKETING",
-  "MURMUR_BUCKETING"
-};
-const std::map 
_BucketingVersion_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kBucketingVersionValues, _kBucketingVersionNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
-
 int _kFunctionTypeValues[] = {
   FunctionType::JAVA
 };
@@ -4957,16 +4945,6 @@ void Table::__set_creationMetadata(const 
CreationMetadata& val) {
 __isset.creationMetadata = true;
 }
 
-void Table::__set_bucketingVersion(const BucketingVersion::type val) {
-  this->bucketingVersion = val;
-__isset.bucketingVersion = true;
-}
-
-void Table::__set_loadInBucketedTable(const bool val) {
-  this->loadInBucketedTable = val;
-__isset.loadInBucketedTable = true;
-}
-
 uint32_t Table::read(::apache::thrift::protocol::TProtocol* iprot) {
 
   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
@@ -5143,24 +5121,6 @@ uint32_t 
Table::read(::apache::thrift::protocol::TProtocol* iprot) {
   xfer += iprot->skip(ftype);
 }
 break;
-  case 17:
-if (ftype == ::apache::thrift::protocol::T_I32) {
-  int32_t ecast223;
-  xfer += iprot->readI32(ecast223);
-  this->bucketingVersion = (BucketingVersion::type)ecast223;
-  this->__isset.bucketingVersion = true;
-} else {
-  xfer += iprot->skip(ftype);
-}
-break;
-  case 18:
-if (ftype == ::apache::thrift::protocol::T_BOOL) {
-  xfer += iprot->readBool(this->loadInBucketedTable);
-  this->__isset.loadInBucketedTable = true;
-} else {
-  xfer += iprot->skip(ftype);
-}
-break;
   default:
 xfer += iprot->skip(ftype);
 break;
@@ -5209,10 +5169,10 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("partitionKeys", 
::apache::thrift::protocol::T_LIST, 8);
   {
 xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, 
static_cast(this->partitionKeys.size()));
-std::vector ::const_iterator _iter224;
-for (_iter224 = this->partitionKeys.begin(); _iter224 != 
this->partitionKeys.end(); ++_iter224)
+std::vector ::const_iterator _iter223;
+for (_iter223 = this->partitionKeys.begin(); _iter223 != 
this->partitionKeys.end(); ++_iter223)
 {
-  xfer += (*_iter224).write(oprot);
+  xfer += (*_iter223).write(oprot);
 }
 xfer += oprot->writeListEnd();
   }
@@ -5221,11 +5181,11 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("parameters", 
::apache::thrift::protocol::T_MAP, 9);
   {
 xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, 
::apache::thrift::protocol::T_STRING, 
static_cast(this->parameters.size()));
-std::map ::const_iterator _iter225;
-for (_iter225 = this->parameters.begin(); _iter225 != 
this->parameters.end(); ++_iter225)
+std::map ::const_iterator _iter224;
+for (_iter224 = this->parameters.begin(); _iter224 != 
this->parameters.end(); ++_iter224)
 {
-  xfer += oprot->writeString(_iter225->first);
-  xfer += oprot->writeString(_iter225->second);
+  xfer += oprot->writeString(_iter224->first);
+  xfer += oprot->writeString(_iter224->second);
 }
 xfer += oprot->writeMapEnd();
   }
@@ -5263,16 +5223,6 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
 xfer += this->creationMetadata.write(oprot);
 xfer += oprot->writeFieldEnd();
   }
-  if (this->__isset.bucketingVersion) {
-xfer += oprot->writeFieldBegin("bucketingVersion", 
::apache::thrift::protocol::T_I32, 17);
-xfer += oprot->writeI32((int32_t)this->bucketingVersion);
-xfer += oprot->writeFieldEnd();
-  }
-  if (this->__isset.loadInBucketedTable) {
-xfer += oprot->writeFieldBegin("loadInBucketedTable",

[4/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out 
b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
index 117ff4a..8cfa113 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
@@ -1,8 +1,8 @@
-PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_small
-POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_small
@@ -23,11 +23,27 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/sm
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS 
TEXTFILE
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_small@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_small@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_small@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_small@ds=2008-04-08
+PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS 
TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_big
@@ -48,22 +64,6 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/bi
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
 PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/00_0' INTO TABLE bucket_big 
partition(ds='2008-04-09')
 PREHOOK: type: LOAD
  A masked pattern was here 
@@ -81,22 +81,6 @@

[3/7] hive git commit: Revert "HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)"

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/3464df45/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp 
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index 615c024..af0fd6b 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -1240,14 +1240,14 @@ uint32_t 
ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 this->success.clear();
-uint32_t _size1050;
-::apache::thrift::protocol::TType _etype1053;
-xfer += iprot->readListBegin(_etype1053, _size1050);
-this->success.resize(_size1050);
-uint32_t _i1054;
-for (_i1054 = 0; _i1054 < _size1050; ++_i1054)
+uint32_t _size1049;
+::apache::thrift::protocol::TType _etype1052;
+xfer += iprot->readListBegin(_etype1052, _size1049);
+this->success.resize(_size1049);
+uint32_t _i1053;
+for (_i1053 = 0; _i1053 < _size1049; ++_i1053)
 {
-  xfer += iprot->readString(this->success[_i1054]);
+  xfer += iprot->readString(this->success[_i1053]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1286,10 +1286,10 @@ uint32_t 
ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
 xfer += oprot->writeFieldBegin("success", 
::apache::thrift::protocol::T_LIST, 0);
 {
   xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, 
static_cast(this->success.size()));
-  std::vector ::const_iterator _iter1055;
-  for (_iter1055 = this->success.begin(); _iter1055 != 
this->success.end(); ++_iter1055)
+  std::vector ::const_iterator _iter1054;
+  for (_iter1054 = this->success.begin(); _iter1054 != 
this->success.end(); ++_iter1054)
   {
-xfer += oprot->writeString((*_iter1055));
+xfer += oprot->writeString((*_iter1054));
   }
   xfer += oprot->writeListEnd();
 }
@@ -1334,14 +1334,14 @@ uint32_t 
ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 (*(this->success)).clear();
-uint32_t _size1056;
-::apache::thrift::protocol::TType _etype1059;
-xfer += iprot->readListBegin(_etype1059, _size1056);
-(*(this->success)).resize(_size1056);
-uint32_t _i1060;
-for (_i1060 = 0; _i1060 < _size1056; ++_i1060)
+uint32_t _size1055;
+::apache::thrift::protocol::TType _etype1058;
+xfer += iprot->readListBegin(_etype1058, _size1055);
+(*(this->success)).resize(_size1055);
+uint32_t _i1059;
+for (_i1059 = 0; _i1059 < _size1055; ++_i1059)
 {
-  xfer += iprot->readString((*(this->success))[_i1060]);
+  xfer += iprot->readString((*(this->success))[_i1059]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1458,14 +1458,14 @@ uint32_t 
ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 this->success.clear();
-uint32_t _size1061;
-::apache::thrift::protocol::TType _etype1064;
-xfer += iprot->readListBegin(_etype1064, _size1061);
-this->success.resize(_size1061);
-uint32_t _i1065;
-for (_i1065 = 0; _i1065 < _size1061; ++_i1065)
+uint32_t _size1060;
+::apache::thrift::protocol::TType _etype1063;
+xfer += iprot->readListBegin(_etype1063, _size1060);
+this->success.resize(_size1060);
+uint32_t _i1064;
+for (_i1064 = 0; _i1064 < _size1060; ++_i1064)
 {
-  xfer += iprot->readString(this->success[_i1065]);
+  xfer += iprot->readString(this->success[_i1064]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1504,10 +1504,10 @@ uint32_t 
ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
 xfer += oprot->writeFieldBegin("success", 
::apache::thrift::protocol::T_LIST, 0);
 {
   xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, 
static_cast(this->success.size()));
-  std::vector ::const_iterator _iter1066;
-  for (_iter1066 = this->success.begin(); _iter1066 != 
this->success.end(); ++_iter1066)
+  std::vector ::const_iterator _iter1065;
+  for (_iter1065 = this->success.begin(); _iter1065 != 
this->success.end(); ++_iter1065)
   {
-xfer +=

hive git commit: HIVE-18368: Improve Spark Debug RDD Graph (Sahil Takiar, reviewed by Rui Li)

2018-02-08 Thread stakiar 
Repository: hive
Updated Branches:
  refs/heads/master 43e713746 -> 1e74aca8d


HIVE-18368: Improve Spark Debug RDD Graph (Sahil Takiar, reviewed by Rui Li)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1e74aca8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1e74aca8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1e74aca8

Branch: refs/heads/master
Commit: 1e74aca8d09ea2ef636311d2168b4d34198f7194
Parents: 43e7137
Author: Sahil Takiar 
Authored: Thu Feb 8 12:45:58 2018 -0800
Committer: Sahil Takiar 
Committed: Thu Feb 8 12:45:58 2018 -0800

--
 .../hadoop/hive/ql/exec/spark/CacheTran.java|  14 +-
 .../ql/exec/spark/LocalHiveSparkClient.java |   6 +
 .../hadoop/hive/ql/exec/spark/MapInput.java |  13 +-
 .../hadoop/hive/ql/exec/spark/MapTran.java  |  17 +-
 .../hadoop/hive/ql/exec/spark/ReduceTran.java   |  17 +-
 .../hadoop/hive/ql/exec/spark/ShuffleTran.java  |  19 ++-
 .../hadoop/hive/ql/exec/spark/SparkPlan.java| 164 ++-
 .../hive/ql/exec/spark/SparkPlanGenerator.java  |  35 +++-
 .../hadoop/hive/ql/exec/spark/SparkTran.java|   2 -
 .../hive/ql/exec/spark/SparkUtilities.java  |  36 +---
 .../hive/ql/io/CombineHiveInputFormat.java  |   2 +-
 11 files changed, 85 insertions(+), 240 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1e74aca8/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
index c5fec7d..4b77ac9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
@@ -27,9 +27,11 @@ public abstract class CacheTran cachedRDD;
+  protected final String name;
 
-  protected CacheTran(boolean cache) {
+  protected CacheTran(boolean cache, String name) {
 this.caching = cache;
+this.name = name;
   }
 
   @Override
@@ -40,9 +42,10 @@ public abstract class CacheTran rdd = doTransform(input);
+  return rdd.setName(this.name + " (" + rdd.getNumPartitions() + ")");
 }
   }
 
@@ -51,4 +54,9 @@ public abstract class CacheTran doTransform(JavaPairRDD 
input);
+
+  @Override
+  public String getName() {
+return name;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/1e74aca8/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index cab97a0..f43b449 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -25,7 +25,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hive.ql.exec.DagUtils;
 import org.apache.hive.spark.client.SparkClientUtilities;
+import org.apache.spark.util.CallSite;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.FileSystem;
@@ -160,8 +162,12 @@ public class LocalHiveSparkClient implements 
HiveSparkClient {
 
 // Execute generated plan.
 JavaPairRDD finalRDD = plan.generateGraph();
+
+sc.setJobGroup("queryId = " + sparkWork.getQueryId(), 
DagUtils.getQueryName(jobConf));
+
 // We use Spark RDD async action to submit job as it's the only way to get 
jobId now.
 JavaFutureAction future = 
finalRDD.foreachAsync(HiveVoidFunction.getInstance());
+
 // As we always use foreach action to submit RDD graph, it would only 
trigger one job.
 int jobId = future.jobIds().get(0);
 LocalSparkJobStatus sparkJobStatus = new LocalSparkJobStatus(

http://git-wip-us.apache.org/repos/asf/hive/blob/1e74aca8/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
index d240d18..b1a0d55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
@@ -36,17 +36,18 @@ public class MapInput implements 
SparkTran hadoopRDD;
   private boolean toCache;
   private final SparkPlan sparkPlan;
-  private String name = "MapInput";
+  private final String

hive git commit: HIVE-18389: Print out Spark Web UI URL to the console log (Sahil Takiar, reviewed by Peter Vary)

2018-02-08 Thread stakiar 
Repository: hive
Updated Branches:
  refs/heads/master b8fdd1392 -> 43e713746


HIVE-18389: Print out Spark Web UI URL to the console log (Sahil Takiar, 
reviewed by Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/43e71374
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/43e71374
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/43e71374

Branch: refs/heads/master
Commit: 43e713746f038fb55e4a63dba59945bbfe448471
Parents: b8fdd13
Author: Sahil Takiar 
Authored: Thu Feb 8 11:32:26 2018 -0800
Committer: Sahil Takiar 
Committed: Thu Feb 8 11:32:26 2018 -0800

--
 .../hive/ql/exec/spark/RemoteHiveSparkClient.java|  3 ++-
 .../org/apache/hive/spark/client/BaseProtocol.java   | 12 
 .../org/apache/hive/spark/client/RemoteDriver.java   | 15 +++
 .../apache/hive/spark/client/SparkClientFactory.java |  8 +---
 .../apache/hive/spark/client/SparkClientImpl.java| 10 +-
 .../apache/hive/spark/client/TestSparkClient.java|  4 +++-
 6 files changed, 46 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/43e71374/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index c571d1a..93d44dc 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -101,7 +101,8 @@ public class RemoteHiveSparkClient implements 
HiveSparkClient {
   }
 
   private void createRemoteClient() throws Exception {
-remoteClient = SparkClientFactory.createClient(conf, hiveConf, sessionId);
+remoteClient = SparkClientFactory.createClient(conf, hiveConf, sessionId,
+SessionState.LogHelper.getInfoStream());
 
 if (HiveConf.getBoolVar(hiveConf, ConfVars.HIVE_PREWARM_ENABLED) &&
 (SparkClientUtilities.isYarnMaster(hiveConf.get("spark.master")) ||

http://git-wip-us.apache.org/repos/asf/hive/blob/43e71374/spark-client/src/main/java/org/apache/hive/spark/client/BaseProtocol.java
--
diff --git 
a/spark-client/src/main/java/org/apache/hive/spark/client/BaseProtocol.java 
b/spark-client/src/main/java/org/apache/hive/spark/client/BaseProtocol.java
index 0930325..7290809 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/BaseProtocol.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/BaseProtocol.java
@@ -166,4 +166,16 @@ abstract class BaseProtocol extends RpcDispatcher {
 
   }
 
+  protected static class SparkUIWebURL implements 
Serializable {
+
+final String UIWebURL;
+
+SparkUIWebURL(String UIWebURL) {
+  this.UIWebURL = UIWebURL;
+}
+
+SparkUIWebURL() {
+  this(null);
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/43e71374/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
--
diff --git 
a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java 
b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
index ede8ce9..66cf339 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
@@ -25,6 +25,8 @@ import io.netty.channel.nio.NioEventLoopGroup;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.URI;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -169,6 +171,14 @@ public class RemoteDriver {
   throw e;
 }
 
+if (jc.sc().sc().uiWebUrl().isDefined()) {
+  // Run a reverse DNS lookup on the URL
+  URI uri = URI.create(jc.sc().sc().uiWebUrl().get());
+  InetAddress address = InetAddress.getByName(uri.getHost());
+  this.protocol.sendUIWebURL(uri.getScheme() + "://" + 
address.getCanonicalHostName() + ":" +
+  uri.getPort());
+}
+
 synchronized (jcLock) {
   for (Iterator it = jobQueue.iterator(); it.hasNext();) {
 it.next().submit();
@@ -270,6 +280,11 @@ public class RemoteDriver {
   clientRpc.call(new JobMetrics(jobId, sparkJobId, stageId, taskId, 
metrics));
 }
 
+void sendUIWebURL(String UIWebURL) {
+  LOG.debug("Send UIWebURL({}) to Client.", UIWebURL);
+  clientRpc.call(new SparkUIWebURL(UIWebURL));
+}
+
 private void

hive git commit: HIVE-18511: Fix generated checkstyle errors (Peter Vary, reviewed by Yongzhi Chen)

2018-02-08 Thread pvary 
Repository: hive
Updated Branches:
  refs/heads/master 6e9b63e48 -> b8fdd1392


HIVE-18511: Fix generated checkstyle errors (Peter Vary, reviewed by Yongzhi 
Chen)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8fdd139
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8fdd139
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8fdd139

Branch: refs/heads/master
Commit: b8fdd1392077c4f6dab685a2b3e87ab2d26ac13e
Parents: 6e9b63e
Author: Peter Vary 
Authored: Thu Feb 8 14:42:43 2018 +0100
Committer: Peter Vary 
Committed: Thu Feb 8 14:42:43 2018 +0100

--
 .../client/MetaStoreFactoryForTests.java|  8 ---
 .../hive/metastore/client/TestDatabases.java|  5 -
 .../hive/metastore/client/package-info.java | 22 +++
 .../minihms/AbstractMetaStoreService.java   | 10 -
 .../minihms/EmbeddedMetaStoreForTests.java  |  2 +-
 .../hadoop/hive/metastore/minihms/MiniHMS.java  | 22 ---
 .../minihms/RemoteMetaStoreForTests.java| 10 +
 .../hive/metastore/minihms/package-info.java| 23 
 8 files changed, 84 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b8fdd139/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java
--
diff --git 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java
 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java
index e2b1a68..e723f60 100644
--- 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java
+++ 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java
@@ -34,9 +34,11 @@ import java.util.List;
  * {@link org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService} 
implementations for
  * tests.
  */
-public class MetaStoreFactoryForTests {
+public final class MetaStoreFactoryForTests {
   private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100;
 
+  private MetaStoreFactoryForTests() {}
+
   /**
* We would like to run the tests with 2 MetaStore configurations
* - Embedded - Where the MetaStore is running in the same thread, and does 
not use Thrift
@@ -94,7 +96,7 @@ public class MetaStoreFactoryForTests {
 .setConf(conf)
 .setType(MiniHMS.MiniHMSType.EMBEDDED)
 .build();
-metaStores.add(new Object[] { "Embedded", embedded});
+metaStores.add(new Object[] {"Embedded", embedded});
 
 // Create Remote MetaStore
 conf.set("javax.jdo.option.ConnectionURL",
@@ -104,7 +106,7 @@ public class MetaStoreFactoryForTests {
 .setConf(conf)
 .setType(MiniHMS.MiniHMSType.REMOTE)
 .build();
-metaStores.add(new Object[] { "Remote", remote});
+metaStores.add(new Object[] {"Remote", remote});
 
 return metaStores;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/b8fdd139/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java
--
diff --git 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java
 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java
index fe7e0e4..db80072 100644
--- 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java
+++ 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java
@@ -52,6 +52,9 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+/**
+ * Test class for IMetaStoreClient API. Testing the Database related functions.
+ */
 @RunWith(Parameterized.class)
 public class TestDatabases {
   private static final Logger LOG = 
LoggerFactory.getLogger(TestDatabases.class);
@@ -408,7 +411,7 @@ public class TestDatabases {
   }
 
   /**
-   * Creates an index in the given database for testing purposes
+   * Creates an index in the given database for testing purposes.
* @param databaseName The database name in which the index should be creatd
* @throws TException If there is an error during the index creation
*/

http://git-wip-us.apache.org/repos/asf/hive/blob/b8fdd139/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/package-info.java
--
diff --git 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/package-info.java

[4/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out 
b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
index 8cfa113..117ff4a 100644
--- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out
@@ -1,8 +1,8 @@
-PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_small
-POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_small
@@ -23,27 +23,11 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/sm
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_small@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_small@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS 
TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS 
TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_big
@@ -64,6 +48,22 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/bi
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
 PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/00_0' INTO TABLE bucket_big 
partition(ds='2008-04-09')
 PREHOOK: type: LOAD
  A masked pattern was here 
@@ -81,6 +81,22 @@

[2/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp 
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index aadf8f1..2169265 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -149,6 +149,18 @@ const char* _kEventRequestTypeNames[] = {
 };
 const std::map 
_EventRequestType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kEventRequestTypeValues, _kEventRequestTypeNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
 
+int _kBucketingVersionValues[] = {
+  BucketingVersion::INVALID_BUCKETING,
+  BucketingVersion::JAVA_BUCKETING,
+  BucketingVersion::MURMUR_BUCKETING
+};
+const char* _kBucketingVersionNames[] = {
+  "INVALID_BUCKETING",
+  "JAVA_BUCKETING",
+  "MURMUR_BUCKETING"
+};
+const std::map 
_BucketingVersion_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, 
_kBucketingVersionValues, _kBucketingVersionNames), 
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
 int _kFunctionTypeValues[] = {
   FunctionType::JAVA
 };
@@ -4945,6 +4957,16 @@ void Table::__set_creationMetadata(const 
CreationMetadata& val) {
 __isset.creationMetadata = true;
 }
 
+void Table::__set_bucketingVersion(const BucketingVersion::type val) {
+  this->bucketingVersion = val;
+__isset.bucketingVersion = true;
+}
+
+void Table::__set_loadInBucketedTable(const bool val) {
+  this->loadInBucketedTable = val;
+__isset.loadInBucketedTable = true;
+}
+
 uint32_t Table::read(::apache::thrift::protocol::TProtocol* iprot) {
 
   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
@@ -5121,6 +5143,24 @@ uint32_t 
Table::read(::apache::thrift::protocol::TProtocol* iprot) {
   xfer += iprot->skip(ftype);
 }
 break;
+  case 17:
+if (ftype == ::apache::thrift::protocol::T_I32) {
+  int32_t ecast223;
+  xfer += iprot->readI32(ecast223);
+  this->bucketingVersion = (BucketingVersion::type)ecast223;
+  this->__isset.bucketingVersion = true;
+} else {
+  xfer += iprot->skip(ftype);
+}
+break;
+  case 18:
+if (ftype == ::apache::thrift::protocol::T_BOOL) {
+  xfer += iprot->readBool(this->loadInBucketedTable);
+  this->__isset.loadInBucketedTable = true;
+} else {
+  xfer += iprot->skip(ftype);
+}
+break;
   default:
 xfer += iprot->skip(ftype);
 break;
@@ -5169,10 +5209,10 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("partitionKeys", 
::apache::thrift::protocol::T_LIST, 8);
   {
 xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, 
static_cast(this->partitionKeys.size()));
-std::vector ::const_iterator _iter223;
-for (_iter223 = this->partitionKeys.begin(); _iter223 != 
this->partitionKeys.end(); ++_iter223)
+std::vector ::const_iterator _iter224;
+for (_iter224 = this->partitionKeys.begin(); _iter224 != 
this->partitionKeys.end(); ++_iter224)
 {
-  xfer += (*_iter223).write(oprot);
+  xfer += (*_iter224).write(oprot);
 }
 xfer += oprot->writeListEnd();
   }
@@ -5181,11 +5221,11 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("parameters", 
::apache::thrift::protocol::T_MAP, 9);
   {
 xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, 
::apache::thrift::protocol::T_STRING, 
static_cast(this->parameters.size()));
-std::map ::const_iterator _iter224;
-for (_iter224 = this->parameters.begin(); _iter224 != 
this->parameters.end(); ++_iter224)
+std::map ::const_iterator _iter225;
+for (_iter225 = this->parameters.begin(); _iter225 != 
this->parameters.end(); ++_iter225)
 {
-  xfer += oprot->writeString(_iter224->first);
-  xfer += oprot->writeString(_iter224->second);
+  xfer += oprot->writeString(_iter225->first);
+  xfer += oprot->writeString(_iter225->second);
 }
 xfer += oprot->writeMapEnd();
   }
@@ -5223,6 +5263,16 @@ uint32_t 
Table::write(::apache::thrift::protocol::TProtocol* oprot) const {
 xfer += this->creationMetadata.write(oprot);
 xfer += oprot->writeFieldEnd();
   }
+  if (this->__isset.bucketingVersion) {
+xfer += oprot->writeFieldBegin("bucketingVersion", 
::apache::thrift::protocol::T_I32, 17);
+xfer += oprot->writeI32((int32_t)this->bucketingVersion);
+xfer += oprot->writeFieldEnd();
+  }
+  if (this->__isset.loadInBucketedTable) {
+xfer += oprot->writeFieldBegin("loadInBucketedTable",

[1/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
Repository: hive
Updated Branches:
  refs/heads/master 1faadb074 -> 6e9b63e48


http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h 
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 4c09bc8..44a2c83 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -132,6 +132,16 @@ struct EventRequestType {
 
 extern const std::map _EventRequestType_VALUES_TO_NAMES;
 
+struct BucketingVersion {
+  enum type {
+INVALID_BUCKETING = 0,
+JAVA_BUCKETING = 1,
+MURMUR_BUCKETING = 2
+  };
+};
+
+extern const std::map _BucketingVersion_VALUES_TO_NAMES;
+
 struct FunctionType {
   enum type {
 JAVA = 1
@@ -2372,7 +2382,7 @@ inline std::ostream& operator<<(std::ostream& out, const 
StorageDescriptor& obj)
 }
 
 typedef struct _Table__isset {
-  _Table__isset() : tableName(false), dbName(false), owner(false), 
createTime(false), lastAccessTime(false), retention(false), sd(false), 
partitionKeys(false), parameters(false), viewOriginalText(false), 
viewExpandedText(false), tableType(false), privileges(false), temporary(true), 
rewriteEnabled(false), creationMetadata(false) {}
+  _Table__isset() : tableName(false), dbName(false), owner(false), 
createTime(false), lastAccessTime(false), retention(false), sd(false), 
partitionKeys(false), parameters(false), viewOriginalText(false), 
viewExpandedText(false), tableType(false), privileges(false), temporary(true), 
rewriteEnabled(false), creationMetadata(false), bucketingVersion(true), 
loadInBucketedTable(true) {}
   bool tableName :1;
   bool dbName :1;
   bool owner :1;
@@ -2389,6 +2399,8 @@ typedef struct _Table__isset {
   bool temporary :1;
   bool rewriteEnabled :1;
   bool creationMetadata :1;
+  bool bucketingVersion :1;
+  bool loadInBucketedTable :1;
 } _Table__isset;
 
 class Table {
@@ -2396,7 +2408,9 @@ class Table {
 
   Table(const Table&);
   Table& operator=(const Table&);
-  Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), 
retention(0), viewOriginalText(), viewExpandedText(), tableType(), 
temporary(false), rewriteEnabled(0) {
+  Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), 
retention(0), viewOriginalText(), viewExpandedText(), tableType(), 
temporary(false), rewriteEnabled(0), 
bucketingVersion((BucketingVersion::type)1), loadInBucketedTable(false) {
+bucketingVersion = (BucketingVersion::type)1;
+
   }
 
   virtual ~Table() throw();
@@ -2416,6 +2430,8 @@ class Table {
   bool temporary;
   bool rewriteEnabled;
   CreationMetadata creationMetadata;
+  BucketingVersion::type bucketingVersion;
+  bool loadInBucketedTable;
 
   _Table__isset __isset;
 
@@ -2451,6 +2467,10 @@ class Table {
 
   void __set_creationMetadata(const CreationMetadata& val);
 
+  void __set_bucketingVersion(const BucketingVersion::type val);
+
+  void __set_loadInBucketedTable(const bool val);
+
   bool operator == (const Table & rhs) const
   {
 if (!(tableName == rhs.tableName))
@@ -2493,6 +2513,14 @@ class Table {
   return false;
 else if (__isset.creationMetadata && !(creationMetadata == 
rhs.creationMetadata))
   return false;
+if (__isset.bucketingVersion != rhs.__isset.bucketingVersion)
+  return false;
+else if (__isset.bucketingVersion && !(bucketingVersion == 
rhs.bucketingVersion))
+  return false;
+if (__isset.loadInBucketedTable != rhs.__isset.loadInBucketedTable)
+  return false;
+else if (__isset.loadInBucketedTable && !(loadInBucketedTable == 
rhs.loadInBucketedTable))
+  return false;
 return true;
   }
   bool operator != (const Table ) const {

http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
 
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
new file mode 100644
index 000..b7de161
--- /dev/null
+++ 
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java
@@ -0,0 +1,48 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.thrift.TEnum;
+
+public enum BucketingVersion implements org.apache.thrift.TEnum {
+  INVALID_BUCKETING(0),
+

[6/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
--
diff --git a/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out 
b/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
index 0d586fd..b54c574 100644
--- a/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
+++ b/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
@@ -72,11 +72,11 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/sm
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_small@ds=2008-04-09
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS 
TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS 
TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_big
@@ -97,6 +97,22 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/bi
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
 PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN 
bucket_big b ON a.key = b.key
 PREHOOK: type: QUERY
 POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN 
bucket_big b ON a.key = b.key
@@ -111,16 +127,16 @@ STAGE PLANS:
   Map Operator Tree:
   TableScan
 alias: b
-Statistics: Num rows: 56 Data size: 27500 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 120 Data size: 58120 Basic stats: COMPLETE 
Column stats: NONE
 GatherStats: false
 Filter Operator
   isSamplingPred: false
   predicate: key is not null (type: boolean)
-  Statistics: Num rows: 56 Data size: 27500 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 120 Data size: 58120 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: key (type: string)
 outputColumnNames: _col0
-Statistics: Num rows: 56 Data size: 27500 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 120 Data size: 58120 Basic stats: 
COMPLETE Column stats: NONE
 Sorted Merge Bucket Map Join Operator
   condition map:
Inner Join 0 to 1
@@ -150,7 +166,7 @@ STAGE PLANS:
 partition values:
   ds 2008-04-08
 properties:
-  bucket_count 2
+  bucket_count 4
   bucket_field_name key
   column.name.delimiter ,
   columns key,value
@@ -158,7 +174,7 @@ STAGE PLANS:
   columns.types string:string
  A masked pattern was here 
   name default.bucket_big
-  numFiles 2
+  numFiles 4
   numRows 0
   partition_columns ds
   partition_columns.types string
@@ -166,7 +182,7 @@ STAGE PLANS:
   serialization.ddl struct bucket_big { string key, string value}
   serialization.format 1
   serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-

[7/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
HIVE-18350 : load data should rename files consistent with insert statements. 
(Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e9b63e4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e9b63e4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e9b63e4

Branch: refs/heads/master
Commit: 6e9b63e48b4f34ba26a6eefb354b0c94ee82256c
Parents: 1faadb0
Author: Deepak Jaiswal 
Authored: Thu Feb 8 00:46:28 2018 -0800
Committer: Deepak Jaiswal 
Committed: Thu Feb 8 00:48:20 2018 -0800

--
 .../hive/ql/exec/tez/CustomPartitionVertex.java |   71 +-
 .../ql/exec/tez/CustomVertexConfiguration.java  |   31 +-
 .../hadoop/hive/ql/exec/tez/DagUtils.java   |   16 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   |   16 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   54 +-
 .../annotation/OpTraitsRulesProcFactory.java|   35 +-
 .../optimizer/spark/SparkMapJoinOptimizer.java  |3 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java |   43 +
 .../apache/hadoop/hive/ql/plan/OpTraits.java|   22 +-
 .../hadoop/hive/ql/metadata/TestHive.java   |9 +-
 .../clientpositive/auto_sortmerge_join_2.q  |   12 +-
 .../clientpositive/auto_sortmerge_join_4.q  |6 +-
 .../clientpositive/auto_sortmerge_join_5.q  |   10 +-
 .../clientpositive/auto_sortmerge_join_7.q  |8 +-
 .../bucket_mapjoin_mismatch1.q.out  |  170 -
 .../clientpositive/auto_sortmerge_join_2.q.out  |  196 +-
 .../clientpositive/auto_sortmerge_join_4.q.out  |  104 +-
 .../clientpositive/auto_sortmerge_join_5.q.out  |  158 +-
 .../clientpositive/auto_sortmerge_join_7.q.out  |  168 +-
 .../llap/auto_sortmerge_join_2.q.out|  132 +-
 .../llap/auto_sortmerge_join_4.q.out|   74 +-
 .../llap/auto_sortmerge_join_5.q.out|  112 +-
 .../llap/auto_sortmerge_join_7.q.out|  114 +-
 .../spark/auto_sortmerge_join_2.q.out   |  104 +-
 .../spark/auto_sortmerge_join_4.q.out   |   74 +-
 .../spark/auto_sortmerge_join_5.q.out   |  118 +-
 .../spark/auto_sortmerge_join_7.q.out   |  114 +-
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  | 2240 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 3888 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   32 +-
 .../hive/metastore/api/BucketingVersion.java|   48 +
 .../apache/hadoop/hive/metastore/api/Table.java |  230 +-
 .../src/gen/thrift/gen-php/metastore/Types.php  |   57 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   45 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   17 +-
 .../hadoop/hive/metastore/ObjectStore.java  |5 +-
 .../hadoop/hive/metastore/model/MTable.java |   37 +-
 .../src/main/thrift/hive_metastore.thrift   |   11 +-
 .../hive/metastore/cache/TestCachedStore.java   |   18 +-
 .../TestTablesCreateDropAlterTruncate.java  |   19 +-
 40 files changed, 4676 insertions(+), 3945 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index 26afe90..ef148d9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -108,14 +108,15 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
   private final Multimap bucketToTaskMap = 
HashMultimap. create();
 
   private final Map> 
inputToGroupedSplitMap =
-  new HashMap>();
+  new HashMap<>();
 
   private int numInputsAffectingRootInputSpecUpdate = 1;
   private int numInputsSeenSoFar = 0;
   private final Map emMap = 
Maps.newHashMap();
   private final List finalSplits = Lists.newLinkedList();
   private final Map inputNameInputSpecMap =
-  new HashMap();
+  new HashMap<>();
+  private Map inputToBucketMap;
 
   public CustomPartitionVertex(VertexManagerPluginContext context) {
 super(context);
@@ -137,6 +138,7 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
 this.mainWorkName = vertexConf.getInputName();
 this.vertexType = vertexConf.getVertexType();
 this.numInputsAffectingRootInputSpecUpdate = vertexConf.getNumInputs();
+this.inputToBucketMap =

[3/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
--
diff --git 
a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp 
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index af0fd6b..615c024 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -1240,14 +1240,14 @@ uint32_t 
ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 this->success.clear();
-uint32_t _size1049;
-::apache::thrift::protocol::TType _etype1052;
-xfer += iprot->readListBegin(_etype1052, _size1049);
-this->success.resize(_size1049);
-uint32_t _i1053;
-for (_i1053 = 0; _i1053 < _size1049; ++_i1053)
+uint32_t _size1050;
+::apache::thrift::protocol::TType _etype1053;
+xfer += iprot->readListBegin(_etype1053, _size1050);
+this->success.resize(_size1050);
+uint32_t _i1054;
+for (_i1054 = 0; _i1054 < _size1050; ++_i1054)
 {
-  xfer += iprot->readString(this->success[_i1053]);
+  xfer += iprot->readString(this->success[_i1054]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1286,10 +1286,10 @@ uint32_t 
ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
 xfer += oprot->writeFieldBegin("success", 
::apache::thrift::protocol::T_LIST, 0);
 {
   xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, 
static_cast(this->success.size()));
-  std::vector ::const_iterator _iter1054;
-  for (_iter1054 = this->success.begin(); _iter1054 != 
this->success.end(); ++_iter1054)
+  std::vector ::const_iterator _iter1055;
+  for (_iter1055 = this->success.begin(); _iter1055 != 
this->success.end(); ++_iter1055)
   {
-xfer += oprot->writeString((*_iter1054));
+xfer += oprot->writeString((*_iter1055));
   }
   xfer += oprot->writeListEnd();
 }
@@ -1334,14 +1334,14 @@ uint32_t 
ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 (*(this->success)).clear();
-uint32_t _size1055;
-::apache::thrift::protocol::TType _etype1058;
-xfer += iprot->readListBegin(_etype1058, _size1055);
-(*(this->success)).resize(_size1055);
-uint32_t _i1059;
-for (_i1059 = 0; _i1059 < _size1055; ++_i1059)
+uint32_t _size1056;
+::apache::thrift::protocol::TType _etype1059;
+xfer += iprot->readListBegin(_etype1059, _size1056);
+(*(this->success)).resize(_size1056);
+uint32_t _i1060;
+for (_i1060 = 0; _i1060 < _size1056; ++_i1060)
 {
-  xfer += iprot->readString((*(this->success))[_i1059]);
+  xfer += iprot->readString((*(this->success))[_i1060]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1458,14 +1458,14 @@ uint32_t 
ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
 if (ftype == ::apache::thrift::protocol::T_LIST) {
   {
 this->success.clear();
-uint32_t _size1060;
-::apache::thrift::protocol::TType _etype1063;
-xfer += iprot->readListBegin(_etype1063, _size1060);
-this->success.resize(_size1060);
-uint32_t _i1064;
-for (_i1064 = 0; _i1064 < _size1060; ++_i1064)
+uint32_t _size1061;
+::apache::thrift::protocol::TType _etype1064;
+xfer += iprot->readListBegin(_etype1064, _size1061);
+this->success.resize(_size1061);
+uint32_t _i1065;
+for (_i1065 = 0; _i1065 < _size1061; ++_i1065)
 {
-  xfer += iprot->readString(this->success[_i1064]);
+  xfer += iprot->readString(this->success[_i1065]);
 }
 xfer += iprot->readListEnd();
   }
@@ -1504,10 +1504,10 @@ uint32_t 
ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
 xfer += oprot->writeFieldBegin("success", 
::apache::thrift::protocol::T_LIST, 0);
 {
   xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, 
static_cast(this->success.size()));
-  std::vector ::const_iterator _iter1065;
-  for (_iter1065 = this->success.begin(); _iter1065 != 
this->success.end(); ++_iter1065)
+  std::vector ::const_iterator _iter1066;
+  for (_iter1066 = this->success.begin(); _iter1066 != 
this->success.end(); ++_iter1066)
   {
-xfer +=

[5/7] hive git commit: HIVE-18350 : load data should rename files consistent with insert statements. (Deepak Jaiswal, reviewed by Sergey Shelukhin and Ashutosh Chauhan)

2018-02-08 Thread djaiswal 
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out 
b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
index 054b0d0..d4472cf 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out
@@ -1,8 +1,8 @@
-PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_small
-POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_small
@@ -23,27 +23,11 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/sm
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_small@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/02_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-PREHOOK: type: LOAD
- A masked pattern was here 
-PREHOOK: Output: default@bucket_small@ds=2008-04-08
-POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/small/03_0' INTO TABLE bucket_small 
partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
- A masked pattern was here 
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS 
TEXTFILE
+PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned 
by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS 
TEXTFILE
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS 
STORED AS TEXTFILE
+POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) 
partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS 
STORED AS TEXTFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@bucket_big
@@ -64,6 +48,22 @@ POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/bi
 POSTHOOK: type: LOAD
  A masked pattern was here 
 POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/02_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
+PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+ A masked pattern was here 
+PREHOOK: Output: default@bucket_big@ds=2008-04-08
+POSTHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/03_0' INTO TABLE bucket_big 
partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+ A masked pattern was here 
+POSTHOOK: Output: default@bucket_big@ds=2008-04-08
 PREHOOK: query: load data local inpath 
'../../data/files/auto_sortmerge_join/big/00_0' INTO TABLE bucket_big 
partition(ds='2008-04-09')
 PREHOOK: type: LOAD
  A masked pattern was here 
@@ -81,6 +81,22 @@